diff --git a/AGENTS.md b/AGENTS.md index b3749fc8..0a0fd4c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -127,7 +127,13 @@ uv run sphinx-build -b html docs dist/docs # to paste, and it has to be code that still works (and type-checks) # - pipeline stage header blocks: each _pipeline/*.py docstring declares # "Reads:". That is checkable, so check it rather than reading it: -# compare it against grep -oE '\b(policy|lexicon)\.[a-z_]+' on the module +# compare it against grep -oE '\b(policy|lexicon|lex)(\._sets)?\.[a-z][a-z_]*' +# on the module (a vocabulary set is read as `lexicon._sets.titles` +# since 4i-a, and a Reads: line names it `Lexicon.titles`). A Reads: +# line may name a set of fields at once: _classify's "every Lexicon +# vocabulary field except surnames and honorific_tails" stands for +# twenty, 17 read in the module (the grep lists them) and the rest +# through _vocab's helpers, so check what it leaves out # - tests/v2/cases.py notes, which explain why a row lands where it does # - docs/design/rules/, decisions/ and mechanisms.md -- READ, don't # grep: the excerpt and example tests catch citation and example drift, @@ -239,12 +245,12 @@ The 2.0 rewrite landed as underscore-private modules beside the v1 code, which 2 - **Per-word vocabulary fields warn on multi-word entries** (`_normset`/`_normpairs` via `_warn_dead_entry`, UserWarning, never a raise — see the given_name_titles Gotcha for why raising is wrong). `given_name_titles` is the one multi-word-matched field and is exempt; `_edit` passes `warn=False` (add() warns once via the new instance's `__post_init__`; remove() stores nothing). The default vocabulary and every locale pack must stay warning-free (`test_default_lexicon_builds_warning_free`, `test_pack_vocabulary_entries_are_single_words`). - **Invariants guard harm, not no-ops**: add a constructor check when violating it produces a *wrong parse*, not when it produces *nothing*. A false positive costs a working configuration; a true positive on an inert condition costs the user nothing, so that trade is never worth taking. `suffix_acronyms_ambiguous ∩ suffix_words` is guarded because the overlap loses a family name; `given_name_titles` is not, because an unreachable entry is simply never consulted (see Gotchas). Before adding one, construct the config it forbids and check what actually breaks. **Between raise and silence sits the construction-time `UserWarning`**, for a gap that is statically decidable, harmless to SOME deliberate caller, and indistinguishable-from-working for everyone else: the segmenterless activation (#337 — `parser_for(locales.JA)` without a segmenter behaved exactly like a working parser minus the feature) warns rather than raises because the inert JA registration is itself a pinned property, and a warning is filterable by the caller who wants exactly that. The message must carry every applicable remedy and no inapplicable one (the `ja_segmenter` hint fires only when a Japanese script is among the dead ones). Test fuzzers that legitimately construct such configs suppress the warning by MESSAGE, never by category — a blanket `UserWarning` ignore would mask the next construction diagnostic (`_quiet_parser` in `tests/v2/test_properties.py` is the pattern). - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). `PolicyPatch`'s repr shows only set (non-UNSET) fields; `_order_repr` must never raise even on an unvalidated patch's garbage `name_order` (PolicyPatch defers validation to apply time); the sweep test in `tests/v2/test_reprs.py` pins that no config repr leaks the UNSET sentinel. -- **Every pipeline stage is one module holding one public function of the same name** (`_tokenize.py`/`tokenize`), with the exceptions named where they stand — `_group.py`'s `marker_run_length`, a shared predicate (mechanisms.md#ONE-PREDICATE-PER-QUESTION), and since #511 `_post_rules.py`'s `suffix_entries`, the R1 entry pass as a function, because `Parser.revise` runs that one pass over a forced-role sub-parse and a stage's tail cannot be called on its own — **and its module docstring declares the contract in three labelled lines**: `Consumes:` what it takes from `ParseState`, `Produces:` what it hands back, `Reads:` which `Policy`/`Lexicon` fields it consults. `Reads:` is the load-bearing one — it makes "which stage do I touch for this feature?" a grep rather than a read-through. The authority for the stage set is `_pipeline/__init__.py`'s `STAGES` (eight, and not public API) with the field-ownership map in `ParseState`'s docstring, pinned by `tests/v2/pipeline/test_state.py`; NOT the rules, which are implementation-free by their own preamble and whose `implemented:` names modules honoring a rule rather than stages. A `_pipeline/` module that is not a stage says so in its first line instead (`_assemble.py`: "Not a stage: …", with a `Reads:` line all the same, since it names a reported field by `Policy.middle_as_family` and a token's reading by three `Lexicon` sets), so absence of the three lines is a claim about the module rather than an oversight. Provenance: §5 of the 2026-07-11 conventions spec, recorded here 2026-08-16. +- **Every pipeline stage is one module holding one public function of the same name** (`_tokenize.py`/`tokenize`), with the exceptions named where they stand — `_group.py`'s `marker_run_length`, a shared predicate (mechanisms.md#ONE-PREDICATE-PER-QUESTION), and since #511 `_post_rules.py`'s `suffix_entries`, the R1 entry pass as a function, because `Parser.revise` runs that one pass over a forced-role sub-parse and a stage's tail cannot be called on its own — **and its module docstring declares the contract in three labelled lines**: `Consumes:` what it takes from `ParseState`, `Produces:` what it hands back, `Reads:` which `Policy`/`Lexicon` fields it consults. `Reads:` is the load-bearing one — it makes "which stage do I touch for this feature?" a grep rather than a read-through. A `Reads:` line names a vocabulary set as `Lexicon.`, and every library module but `_lexicon` reads it as `lexicon._sets.` (`_lexicon._LexiconSets`, #754's 4i-a), never off the public field, `surnames` and `given_names` included though they stay public, so that 4i-b can make the other twenty internal without touching a reader; `tests/v2/test_layering.py` refuses a direct read, field by field, in the library and in every tool. The authority for the stage set is `_pipeline/__init__.py`'s `STAGES` (eight, and not public API) with the field-ownership map in `ParseState`'s docstring, pinned by `tests/v2/pipeline/test_state.py`; NOT the rules, which are implementation-free by their own preamble and whose `implemented:` names modules honoring a rule rather than stages. A `_pipeline/` module that is not a stage says so in its first line instead (`_assemble.py`: "Not a stage: …", with a `Reads:` line all the same, since it names a reported field by `Policy.middle_as_family` and a token's reading by three `Lexicon` sets), so absence of the three lines is a claim about the module rather than an oversight. Provenance: §5 of the 2026-07-11 conventions spec, recorded here 2026-08-16. - **Frames are a budget for hot and scaling paths; everywhere else, prefer the shared predicate.** The parser's cost is counted in Python frames (`sys.setprofile` "call" events — deterministic and load-immune, which is why #475 chose them over wall-clock timing), but one frame is small: measured 2026-09-28 on py3.11, a trivial call costs about 13 ns against 40–80 µs for a whole parse, so one frame per parse is roughly 0.03% of it. What the budget exists to catch is how a cost SCALES and how it ACCUMULATES, so weigh an extra frame by four questions, in order. (1) Does it multiply with the input — per token, per member, per pair? Then it matters, and a hand-inlined fast path can earn its place (the inline tag tests ahead of `_pieces.starts_a_credential_run` in its loops, each a necessary half of the predicate so nothing can drift; `_vocab.run_word_fold` was the measured example until #613 deleted the run test it served). Folding a copy into a shared helper is where such a pre-check gets lost: #617 replaced `_comma._title_chains`, whose loop tested the conjunction tag inline, with a call into group's loop, which asks `is_conj_piece` per piece, and every family-comma parse paid two frames per word of the part (`Smith, John` +2) while the reference row fell -- review caught it; the fix gates the call on the tag walk the caller already makes, and `test_a_comma_part_with_no_free_connective_skips_the_join` holds it. Count a comma name, not only the reference row, when a fold touches the comma path. (2) How many names pay it — every parse, where plain names like `Smith, John` must keep their cost, or only a rare shape? (3) Does it move an enforced guard — the ±2% reference band in `test_benchmark.py` (`_CALL_BASELINE`, decisions/parse-cost) or a scaling ratio? (4) What does the inline alternative cost to maintain? A hand copy of a predicate that can drift is the defect class mechanisms.md#ONE-PREDICATE-PER-QUESTION names, and it compounds; a NARROWER question stated exactly is not a copy and may stay inline on that ground alone. A constant frame on a rare shape that no guard sees loses to readability: #544's cleanup declined `is_title_piece` for one frame on comma credential runs (24 ns a parse, no guard moved), a stricter call than this rule asks for; by this rule the inline test is still right, but because it asks the narrower question (a lone word in both vocabularies), not because of the frame. The converse holds too: the frame counter cannot see C-level work (`x in some_list`, slices, string concatenation), so "no new frames" is not "no new cost" — #553's quadratic was a list membership test inside a loop, invisible to every frame guard. - **A claim about WHICH STAGE or WHICH LAYER does something is checkable — check it before writing it.** The pipeline is eight stages with a written ownership map (`ParseState`'s docstring, pinned by `tests/v2/pipeline/test_state.py`), and `parse(s).tokens` prints every token's role and tags, so "extract assigns this", "classify never sees that", "group consumes it" each have a one-command answer. #329's prose claimed delimited maiden content is *"claimed whole before classify has tagged anything inside it"*; measured, `classify` tags the marker fine and only the CONSUMING is missing, because `_group`'s rule walks `pieces` and a token that already carries a role is not in `pieces`. Two different mechanisms, one plausible sentence covering both. That single claim then shipped SIX times across three correction rounds, which is the part worth internalizing: **when a mechanism claim turns out wrong, sweep for where else you wrote it, and sweep again at the END of the change over the words the change itself just added.** Prose density here means one idea lives in a docstring, a case note, a release-log entry and this file at once; the implementer working against a wrong mechanism is the person most likely to restate it; and rewriting a mechanism claim is writing one, so the correction earns the same one-command check as the original — two of the six instances were fresh errors introduced by the sentence fixing the previous one. **Adjacency is the trap.** The claim that feels already-known is the one about the neighbouring stage or the neighbouring layer: one comment block in the 2.x `tests/v2/test_facade_cases.py` got the exception type, the raising layer, the skip mechanism, the count of skipped rows, and which row was blocked all wrong at once — every one a claim about `_config_shim` versus real 1.4.0, written from reasoning, in the file whose whole job is translating between them. What finally held was not better prose but moving the claim into a test (`_CORE_ONLY_IDS`, deleted with the facade in 3.0), which cannot be wrong the way a sentence can. - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. **Document the positive direction of a partial property**: "a non-empty `ambiguities` is a signal to act on" is checkable, while "an empty one means no fork occurred" is a universal negative needing exhaustive verification -- that claim was written twice and falsified twice, at sites the author had not audited. - **The segmenter contract**: the optional `Parser(segmenter=...)` hook is parse-totality's ONE exception (locales spec section 4). Everything inside that exception is a bug in USER CODE, never a fact about the name, so it is surfaced rather than absorbed: the segmenter's own exceptions propagate, and the two protocol violations the stage can detect for itself — an answer of the wrong type, and one cutting at or past the end of the token it was handed — raise `TypeError`/`ValueError` from `_script_segment` for the same reason. The line to hold when adding a check there: a protocol violation by the segmenter's AUTHOR raises, while an adapter's defense against its own third-party library (`locales/ja.py`'s repertoire, length, reconstruction and score guards) declines with `None`, because what those catch is a fact about the content. -- **Pickling**: v2 types must round-trip (`Parser` is picklable by construction, and it holds a `Lexicon`; the one qualifier is that a `Parser` pickles iff its segmenter does — see the segmenter bullet above). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, with one widening: a field added after its type shipped carries `metadata={_types.ADDED_IN: ""}` and a default, and an older pickle missing it loads with the default (`Ambiguity.alternative`, 2.7; `Token.reading`, `Reading.languages`, `Reading.second_surname` and `ParsedName.order`, 3.0.0) -- a missing field without the marker, default or not, an unknown field and a renamed one are still refused, each a row in `tests/v2/test_types.py::_REFUSED_SKEWS`, so a new field on a pickled type takes the marker or breaks every older pickle; the shared guard deliberately does NOT re-validate values (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. `Lexicon`'s copy of the guard admits the same marker (since 3.0, #959's prerequisite), read under `_lexicon._ADDED_IN`, the same string, since layering keeps `_lexicon` from importing `_types` (`test_lexicon_marks_an_added_field_with_the_shared_key` holds the two equal), so a new `Lexicon` field takes the marker and a default or breaks older `Lexicon`, `Parser` and locale-pack pickles, which `tests/v2/data/lexicon_v28.pickle` holds. An older pickle loads the field's DATACLASS default, not the words a later `Lexicon.default()` compiles into it, so a marked field the default vocabulary fills makes a 2.8 `Parser()` pickle parse differently from the release's `Parser()`, in silence: give such a field a default an older pickle should mean, or record the accepted drift with it (decisions/A1, 2026-10-30; #959's field, filled only by a pack, is the case it does not reach). And unlike the shared guard it DOES re-validate, the loaded default with the rest: it re-runs `__post_init__` and refuses a state holding an entry that is not already normalized, because `particles_ambiguous` changed meaning between v1 and v2 without changing its name, and an old blob would otherwise load an inverted lexicon in silence. +- **Pickling**: v2 types must round-trip (`Parser` is picklable by construction, and it holds a `Lexicon`; the one qualifier is that a `Parser` pickles iff its segmenter does — see the segmenter bullet above). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, with one widening: a field added after its type shipped carries `metadata={_types.ADDED_IN: ""}` and a default, and an older pickle missing it loads with the default (`Ambiguity.alternative`, 2.7; `Token.reading`, `Reading.languages`, `Reading.second_surname` and `ParsedName.order`, 3.0.0) -- a missing field without the marker, default or not, an unknown field and a renamed one are still refused, each a row in `tests/v2/test_types.py::_REFUSED_SKEWS`, so a new field on a pickled type takes the marker or breaks every older pickle; the shared guard deliberately does NOT re-validate values (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. `Lexicon`'s copy of the guard admits the same marker (since 3.0, #959's prerequisite), read under `_lexicon._ADDED_IN`, the same string, since layering keeps `_lexicon` from importing `_types` (`test_lexicon_marks_an_added_field_with_the_shared_key` holds the two equal), so a new `Lexicon` field takes the marker and a default or breaks older `Lexicon`, `Parser` and locale-pack pickles, which `tests/v2/data/lexicon_v28.pickle` holds. An older pickle loads the field's DATACLASS default, not the words a later `Lexicon.default()` compiles into it, so a marked field the default vocabulary fills makes a 2.8 `Parser()` pickle parse differently from the release's `Parser()`, in silence: give such a field a default an older pickle should mean, or record the accepted drift with it (decisions/A1, 2026-10-30; #959's field, filled only by a pack, is the case it does not reach). `Lexicon`'s one slot outside its fields, `_sets` (4i-a), inherited from a plain base so that `dataclasses.fields` and the pickled state stay its fields, is built by `__post_init__` and never pickled, a load rebuilding it. And unlike the shared guard it DOES re-validate, the loaded default with the rest: it re-runs `__post_init__` and refuses a state holding an entry that is not already normalized, because `particles_ambiguous` changed meaning between v1 and v2 without changing its name, and an old blob would otherwise load an inverted lexicon in silence. - **One sanctioned global**: the (future) cached default `Parser`. Lazily cached FROZEN singletons (`Lexicon.default()`'s `functools.cache`, the future default parser) are constants, not state; any second piece of module-level MUTABLE state requires amending the conventions doc, on purpose, in review. - **Tests**: all v2 tests live in the `tests/v2/` package, one test module per source module plus the cross-cutting ones (`test_reprs.py`, `test_layering.py`, `test_contracts.py`, `test_properties.py`, `test_benchmark.py`, the `cases.py`/`test_cases.py` table, `test_regex_sync.py`, which pins the pipeline's hand-copied patterns against their sources, and `test_snapshot.py`, which holds every corpus name's reading to `tools/differential/snapshot.jsonl` — sharing `_differential_fixtures.py` with `test_differential.py` and `test_corpora.py` for the handles on `tools/differential/`), names stating behavior; the 1.x suite's parsing tests stand ported to `parse()` in `tests/*.py`. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. @@ -254,7 +260,7 @@ The 2.0 rewrite landed as underscore-private modules beside the v1 code, which 2 **Before adding a short/common word to `PARTICLES` globally**, test it mid-string against realistic 3-token names, not just check for English-word collisions: Korean/Vietnamese given names put a short syllable in the middle slot (`Park In Hwan`, `Nguyen To Nga`), and Western names put a bare initial there (`John V. Smith`). A word that looks safe ("nobody is named 'to'") can still swallow a real middle name/initial into the last name once it's a global prefix — confirmed regressions for `to`/`in`/`an`/`ten`/`then` and bare `v` this way (PR #191). -**Adding a curated sub-set of an existing vocabulary set** (a marker over a base set, as `AFTER_GIVEN_TITLES` is over `TITLES`) — it is a behavioral PROPERTY of a reading, not a second word list: a row in `nameparser/_reading_index.py`'s `BEHAVIORAL` table, a branch in its `compile_readings` adding the words whose reading carries it and a key in its `Compiled` TypedDict, a constant of the same name in `tools/vocab/build.py`'s `CONSTANTS` roster (an import-time assert there holds the two to one list), and the `NAME = _compiled.NAME` assignment in the config module. A property the compiler reads is kept in the reading index by `indexed()`, which reads `BEHAVIORAL`, so a new one reaches a choice of languages too. The subset relation then holds by construction, since the property cannot exist without its reading, so it needs no check. A marker that records a clash with a NAME is the other shape: derived from the word's name reading beside the clashing one (`SUFFIX_ACRONYMS_AMBIGUOUS`, `SUFFIX_WORDS_AMBIGUOUS`, and the complement `NON_GIVEN_NAME_PARTICLES` works from), not written as a property, so it too holds by construction (decisions/vocabulary-model, 2026-10-25). A relation construction does not give (`∩ == ∅` with a set the sub-set cannot logically overlap; a sub-set member that is also a title is silently inert, title handling consuming it first) goes in `check_pool`, with a mutation in `tests/v2/test_vocab.py`'s table. Do **not** forbid `titles ∩ particles` — that overlap is intentional (`freiherr`, `st`). A caller's `Lexicon.add()` to a sub-set does **not** add the word to its base, so a caller adding a brand-new word adds it to both. Two traps a gating set met (#752 phase 2i). Where a caller listing a word in the BASE set has always meant "read it as this everywhere" (the 1.4 recipe `c.titles.add("dean")`), the gated words compile OUT of the base rather than into both, or the gate overrides the caller's listing: `TITLES_AMBIGUOUS` is not a subset of `TITLES` for that reason, and a word in both reads as a title. And `_VOCAB_FIELDS` is read whole by two rosters in `_pipeline/_vocab.py`, the capitals shape's "unlisted" (`in_any_wordlist`) and the comma contrast's "claimed as non-name" (`_NON_NAME_FIELDS`), so a new field joins both by default; a field whose words may be names leaves both, which the grid found as `John Smith, DEAN` reversed where C6 had read a credential; #789's `given_name_abbreviations` met it again (`John Smith, JAS` and `Jas SMITH, XYZ` reversed), found by review because its grid held only the dotted spellings the set was for -- a new field's grid holds the spellings it must NOT move too. +**Adding a curated sub-set of an existing vocabulary set** (a marker over a base set, as `AFTER_GIVEN_TITLES` is over `TITLES`) — it is a behavioral PROPERTY of a reading, not a second word list: a row in `nameparser/_reading_index.py`'s `BEHAVIORAL` table, a branch in its `compile_readings` adding the words whose reading carries it and a key in its `Compiled` TypedDict, a constant of the same name in `tools/vocab/build.py`'s `CONSTANTS` roster (an import-time assert there holds the two to one list), and the `NAME = _compiled.NAME` assignment in the config module. A property the compiler reads is kept in the reading index by `indexed()`, which reads `BEHAVIORAL`, so a new one reaches a choice of languages too. The subset relation then holds by construction, since the property cannot exist without its reading, so it needs no check. A marker that records a clash with a NAME is the other shape: derived from the word's name reading beside the clashing one (`SUFFIX_ACRONYMS_AMBIGUOUS`, `SUFFIX_WORDS_AMBIGUOUS`, and the complement `NON_GIVEN_NAME_PARTICLES` works from), not written as a property, so it too holds by construction (decisions/vocabulary-model, 2026-10-25). A relation construction does not give (`∩ == ∅` with a set the sub-set cannot logically overlap; a sub-set member that is also a title is silently inert, title handling consuming it first) goes in `check_pool`, with a mutation in `tests/v2/test_vocab.py`'s table. Do **not** forbid `titles ∩ particles` — that overlap is intentional (`freiherr`, `st`). A caller's `Lexicon.add()` to a sub-set does **not** add the word to its base, so a caller adding a brand-new word adds it to both. Two traps a gating set met (#752 phase 2i). Where a caller listing a word in the BASE set has always meant "read it as this everywhere" (the 1.4 recipe `c.titles.add("dean")`), the gated words compile OUT of the base rather than into both, or the gate overrides the caller's listing: `TITLES_AMBIGUOUS` is not a subset of `TITLES` for that reason, and a word in both reads as a title. And `_VOCAB_FIELDS` is read by three rosters: two in `_pipeline/_vocab.py`, the capitals shape's "unlisted" (`in_any_wordlist`, over `_LISTED_FIELDS`, the roster less four fields) and the comma contrast's "claimed as non-name" (`_NON_NAME_FIELDS`, those less `surnames` and `bound_given_names`), and `_assemble._listed_whole`, which reads it whole to tell a word the parse's own lexicon lists for `Reading.languages`. A new field joins all three by default; a field whose words may be names leaves the two in `_vocab`, which the grid found as `John Smith, DEAN` reversed where C6 had read a credential; #789's `given_name_abbreviations` met it again (`John Smith, JAS` and `Jas SMITH, XYZ` reversed), found by review because its grid held only the dotted spellings the set was for -- a new field's grid holds the spellings it must NOT move too. **Validating a new parsing rule** — before implementing, simulate it in a throwaway script against `TEST_NAMES` (plus a few target-language examples) to catch regressions/false-positives early. E.g. this surfaced the `last_base` `do`/`st`/`mc` empties and the patronymic `"David Michael Abramovich"` false-reorder. After the change, `tests/v2/test_snapshot.py` lists every corpus name it moved, grouped by changed field -- read them, and re-record with `uv run python tools/differential/snapshot.py --record` once every move is intended, so the PR diff of `snapshot.jsonl` shows the reviewer the same list. `uv run python tools/differential/cldr_agreement.py` reports how the tree's agreement with CLDR's stated name fields moved; `tests/v2/test_cldr_agreement.py` fails on any move, and `tools/differential/cldr_agreement.json` doubles as a measured list of names the parser and CLDR read differently. The same command and test score the GND corpus's real names (`gnd_fields.jsonl`, record `gnd_agreement.json`) by the same rules, and `--record` rewrites every record, printing per source what it accepts. A locale-pack change reads the second view the same command prints, per pack (`cldr_agreement_locales.json`, #672, and `wikidata_native_agreement_locales.json` for real Han and kana names); its `JA` rows run only with the `ja` extra installed. A change to Han, Hangul or kana order or to Korean segmentation reads `wikidata_native_agreement.json` too: the only real names in those scripts the corpora hold. It holds no honorific, so a change to the honorific peel moves nothing there. A change to comma or credential handling also reads the local corpus of hand-typed names (`snapshot.py --local ctgov-officials --base `, which reads the base from a baseline kept for its package's files and never the shared snapshot beside the corpus (a base read once from the same fetch is copied rather than parsed again; `--sample N` reads a fixed sample while iterating; the shared snapshot beside the corpus is one file per source for every worktree, and #752's phase 2f found ctgov reporting a sibling branch's 1,334 joined-credential readings as its own moves against it); tools/differential/README.md's "Local corpora"): the one population of names nobody chose for a test or a catalog, so it shows how often each shape is typed, and the one no CI run sees. A change to title handling reads `ukec-individuals` the same way: UK donors' names in spoken order and mixed case, in 8,219 of whose 15,214 `parse()` read a title on 2026-10-22. A change to English titles, honorifics or post-nominals also reads `snapshot.py --local govuk-people` the same way: GOV.UK's display names of ministers, officials and officers, the title run and the post-nominals as an office writes them (`The Rt Hon`, `Sir`, `Baroness`, ranks; `KC`, `CBE`, `KCB`), in 2,093 of whose 6,021 `parse()` read one or both on 2026-10-30. A change to Spanish titles, to the `es` pack, or to how a title marks where a name starts after an office reads `snapshot.py --local dof-signatures` the same way: the names in the signature blocks of Mexico's Diario Oficial for 2025, cut from the clause that names the signatory's office, 65.4% of its 33,565 signatures opening with a title (`Lic.`, `Dr.`, `Mtro.`, `C.`, `Licenciada`, `C.P.`), whose first word `parse()` read as a title in 82.1% and `parser_for(locales.ES)` in 86.8% on 2026-10-31. A change to particles or generational words reads `snapshot.py --local tse-donors` the same way, Brazilian names in spoken order dense with both. A change to initials, to a title word outside English, or to how a script's combining marks are read reads `snapshot.py --local nepal-candidates` the same way: Nepal's local election candidates, each a romanized name (surnames written as initials, `B.K.`, `K.C.`; honorific titles such as `Guru` in front, `Shekh`, a title there until #752's phase 2j made it one only inside a title phrase, and `Md.`, which its period makes a title by shape) beside the same name in Devanagari, written with vowel signs and the virama, combining marks the letter-shape tests do not count as letters (decisions/nfd-shapes), and with the joiners ZWJ and ZWNJ. It is the only large population of South Asian names the corpora hold, and its corpus reads both scripts. A change to non-Latin scripts or to Arabic-script names reads `snapshot.py --local jrc-names` the same way, and `local_corpora.py groups jrc-names`: the spellings of one person across scripts and transliterations, harvested from news text, with 17,074 Arabic-script spellings where the committed corpora hold 315 distinct names with an Arabic letter, 96 of them CLDR's fictional renderings (2026-10-24), and under a licence that keeps even a reading of one of its names in the cache. A change to which field a word lands in also reads `local_agreement.py psc-individuals --base ` the same way: real names with the four boxes their filers typed them into (UK companies' people with significant control), scored field by field and by nationality; and `local_agreement.py orcid-researchers --base `, ORCID holders' own two boxes (given names, family name) from every country, scored by the country they work or live in -- a population each person split for themselves, academic but worldwide, the one to read for a change a language pack makes; and `local_agreement.py patentsview-inventors --base `, US patents' inventors in the two boxes their filings give, split by a professional filer, with how many people filed under each name in each country, the one to read for how common a name's word is among a country's US-patent inventors. A change to titles or post-nominals, or to the `cs` pack, reads `local_agreement.py cz-candidates --base `, and with `--locale cs` too: Czech election candidates' four boxes (the title before the name, given names, surname, the title after it), scored by election, the only population whose titles are stated apart from the name, 36% of its 170,181 records titled (2026-10-30), nearly all academic degrees; its names are composed from the boxes (`Ing. Jan Novák, Ph.D.`), so it says nothing about how a writer spaces or punctuates them. These are the only real fielded populations outside the committed fields files, and no CI run sees them either. diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index 3a6bc5b0..d4f4ab9e 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -486,3 +486,10 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) 6. **Migration.** The old spellings (`add`/`remove` with a set keyword, `Lexicon(...)` and `dataclasses.replace` with one, a set attribute read) fail loudly with a per-word translation, the one the pickle load uses. 7. **Work order.** 4i-a, the pipeline reading the sets through one internal object; 4i-b, the break; 4i-c, the recorded choice; 4i-d, `Reading.languages` following the choice, optional; none moves a default parse. - Declined: letting the caller's reading win where it clashes with a reading a later choice turns on (the clash is refused at construction, and one rule at both points is the one a caller can predict); an eager default base, holding the index rows in `Lexicon.default()` (it would reverse the 2026-10-29 lazy load for every caller, about 2 ms cached and 18 to 20 ms uncached on the first use); refusing every 2.x `Lexicon` state, pickle option (a) (the default and the 2.8 fixtures translate back exactly, and the commonest customization, `Lexicon.default().add(titles={"dean"})`, does with a word in both `titles` and `titles_ambiguous` compared as in `titles` alone, the two spellings parsing alike). The alternatives the proposal weighed for its other items, each recorded there with its reason, are declined with them: a positional `add(word, kind)`, a reading object, new method names, publishing `IndexReading`, a predicate per category, recording no choice, a `Parser` field for it, and pickling the base by reference. +- **2026-11-02 #754, sub-phase 4i-a — THE PARSE READS THE VOCABULARY SETS THROUGH ONE INTERNAL OBJECT, `Lexicon._sets`, NOT THE PUBLIC FIELDS; THE THREE TOOLS MOVE TO IT; NOTHING PUBLIC CHANGES AND NO PARSE MOVES.** Carries out item 7's first PR of the 2026-11-02 #754, the `Lexicon` surface entry. + - **What.** `_lexicon._LexiconSets`, a frozen, slotted object with one frozenset per `_VOCAB_FIELDS` entry: the 20 fields 4i-b makes internal, and `surnames` and `given_names`, which stay public but sit in the rosters beside them (`_assemble._listed_whole` walks all of `_VOCAB_FIELDS`, and `_vocab.in_any_wordlist` holds `surnames`; `claimed_as_non_name` holds neither), so that a roster is read off one object. Each `Lexicon` builds one in `__post_init__`, from its normalized fields by keyword (tests/v2/test_lexicon.py holds its field names to `_VOCAB_FIELDS`, in order, so a field added to one and not the other fails), into a slot `_sets` inherited from a plain base class (`_HoldsSets`): a dataclass field, even `init=False`, would have changed `dataclasses.fields(Lexicon)`, the pickled state and the 2.8 pickles' layout check. It is not compared, hashed or pickled; a load, `dataclasses.replace` and `|` rebuild it. Every library module but `_lexicon` reads `lexicon._sets.titles` where it read `lexicon.titles`; a membership test stays a C-level attribute read, one more attribute load and no frame. + - **Measured: the reads,** on this branch over `b55d4d33`, by the 2026-11-01 entry's recipe (an attribute load of a set field, `surnames` and `given_names` left out): its 111 library reads in 12 files are 8 of `_lexicon`'s own, which stay, 2 `TailRead.titles` reads in `_assign` and `_group`, no lexicon's, and 101 that moved, in 11 files. Moved beside them: 5 reads of `surnames` and `given_names`, and 5 `getattr` reads over a roster (the three above and `_parser._apply_choice`'s two). The tools' 16 `Lexicon` reads moved to `_sets`, none to the index: `local_agreement` reads a pack parser's lexicon, which the shipped index does not describe, and `name_usage` reads `suffix_acronyms_ambiguous`, a marker the compile derives and no index row states. The 17th is `local_corpora`'s counter. + - **Measured: the frames,** Python 3.11.17, the benchmark's counter over 50 names with a four-digit counter, `b55d4d33` against this branch: the reference row 311.82 and 311.82 (`uv run python tools/perf/call_count.py --against b55d4d33`); by the same counter, `Smith{i:04d}, John` 168.0 and 168.0, `de la Vega{i:04d}, Juan Q., Jr.` 334.16 and 334.16, `John Smith{i:04d}` 158.0 and 158.0. Wall clock over 6,000 names moved within the run-to-run spread (96 to 98 µs a name on both trees). `_CALL_BASELINE` is unchanged. + - **The guard.** tests/v2/test_layering.py walks every library module but `_lexicon`, and every file under tools/, and refuses a vocabulary field read off a lexicon, one test per `_VOCAB_FIELDS` entry: in the library only a read through `._sets`, or through a name `sets` that every binding in the module assigns from a `_sets`, passes, `TailRead`'s `read.titles` named as no lexicon, and a `getattr` of a lexicon, or of a field named as a constant off anything but a `_sets`, fails; in the tools, whose counters reuse the field names, a read off `lexicon`, `lex`, `.lexicon`, `Lexicon.default()` or a name any binding gives one of those (an assignment, a walrus, a `Lexicon` annotation) fails, the AGENTS.md recipe's `L = Parser().lexicon` among them. Not followed, and stated in the check's docstring: a lexicon handed through a container, a call's return or an attribute other than `.lexicon`, and aliases per scope rather than per module. Its negative controls: a synthetic read in each spelling, library and tool, per field, must be refused, each binding form of `sets` and of an alias has a row, and `_UNGUARDED_READS` records what the check refuses on today's sources with `._sets` taken out again (114 in the library: 106 attribute loads, 5 roster `getattr`s, 3 of them through the `sets` the undoing makes a lexicon, and 3 `sets =` bindings; 16 in the tools, of 40 files). Mutated on the built tree, one `lex._sets.titles_ambiguous` in `_classify` written back as `lex.titles_ambiguous` failed exactly `test_no_reader_takes_a_vocabulary_field_off_a_lexicon[titles_ambiguous]`. + - **How 4i-b plugs in.** 4i-b changes how `_sets` is filled and nothing that reads it: from the lexicon's readings, compiled by `compile_readings`, rather than from its public fields; the default's from nameparser/config/_compiled.py, as `_default_lexicon` reads it today, so `parse("John Smith")` still loads no index (the base stays lazy, the 2026-11-02 entry's item 4). The derived markers the views will read, a listed suffix word's kind and the patronymic particles (the RFC's item 1), join `_LexiconSets` as attributes, and `_vocab.suffix_kind` and classify's particle kind read them there in place of the index. With `dataclasses.fields(Lexicon)` changing in 4i-b anyway, `_sets` may become a private field then, the base class going. `_parser._apply_choice` reads `_sets` but still writes the public fields through `dataclasses.replace`; 4i-b gives it the readings to write until 4i-c retires it. What 4i-b also breaks, outside the readers the guard holds: the tests' 157 public-field reads (the 2026-11-01 count), and AGENTS.md's recipe for the titles in no suffix set, `L.titles - L.suffix_acronyms - L.suffix_words` on `L = Parser().lexicon`, which reads public fields and is rewritten then over `readings_of` or `_sets`. + - The default does not move: tests/v2/test_snapshot.py passes unchanged and tests/v2/test_cldr_agreement.py with it; the suite passes on Python 3.11 and 3.15 without the `ja` extra. diff --git a/docs/design/vocabulary-model-rfc.md b/docs/design/vocabulary-model-rfc.md index a59aebd2..cde5af8a 100644 --- a/docs/design/vocabulary-model-rfc.md +++ b/docs/design/vocabulary-model-rfc.md @@ -463,7 +463,7 @@ How a call reaches a pack's pattern is open, and each answer costs something: `P ### The `Lexicon` surface: configuring by reading -Status: **decided 2026-11-02 as recommended below (decisions/vocabulary-model's 2026-11-02 #754, the `Lexicon` surface entry), item 1's title warning on 2026-11-01 (decisions/vocabulary-model's 2026-11-01 #754, the title warning entry); nothing is built.** What is decided is the direction (2026-10-28): a caller configures vocabulary by reading, `Lexicon` derives the sets and gates the build derives, and the compiled sets become internal (on 2026-11-01, 20 of `Lexicon`'s 23 public fields: the 19 of `_lexicon._COMPILED_FIELDS` and the derived `particles_ambiguous`; item 3 keeps the other three), with no 2.x warning (2026-10-29). This section proposes the API that direction leaves open, each question with the options weighed and one recommendation; decisions/vocabulary-model's 2026-11-01 #754, the `Lexicon` surface proposed entry records the proposal and its measurements, and decisions/vocabulary-model's 2026-11-02 #754, the `Lexicon` surface entry the decision, each recommendation accepted and the alternatives below declined. Where this section says "proposed" or "recommended", read "decided"; the wording is kept as the record of what was weighed. Every parse below was run on master `a3b1f3e1` (Python 3.11.17) through a throwaway prototype that compiles a lexicon's readings with the shipped `compile_readings` over `choose(index(), chosen)`, the whole index with its readings off by default, as item 4 has the default hold them, the caller's edits applied after the choice, and folds the result into today's `Lexicon`, so the real parser reads it; the prototype is not committed. +Status: **decided 2026-11-02 as recommended below (decisions/vocabulary-model's 2026-11-02 #754, the `Lexicon` surface entry), item 1's title warning on 2026-11-01 (decisions/vocabulary-model's 2026-11-01 #754, the title warning entry); 4i-a is built (decisions/vocabulary-model's 2026-11-02 #754, sub-phase 4i-a entry), the rest is not.** What is decided is the direction (2026-10-28): a caller configures vocabulary by reading, `Lexicon` derives the sets and gates the build derives, and the compiled sets become internal (on 2026-11-01, 20 of `Lexicon`'s 23 public fields: the 19 of `_lexicon._COMPILED_FIELDS` and the derived `particles_ambiguous`; item 3 keeps the other three), with no 2.x warning (2026-10-29). This section proposes the API that direction leaves open, each question with the options weighed and one recommendation; decisions/vocabulary-model's 2026-11-01 #754, the `Lexicon` surface proposed entry records the proposal and its measurements, and decisions/vocabulary-model's 2026-11-02 #754, the `Lexicon` surface entry the decision, each recommendation accepted and the alternatives below declined. Where this section says "proposed" or "recommended", read "decided"; the wording is kept as the record of what was weighed. Every parse below was run on master `a3b1f3e1` (Python 3.11.17) through a throwaway prototype that compiles a lexicon's readings with the shipped `compile_readings` over `choose(index(), chosen)`, the whole index with its readings off by default, as item 4 has the default hold them, the caller's edits applied after the choice, and folds the result into today's `Lexicon`, so the real parser reads it; the prototype is not committed. **1. Configuring: `add` and `remove` take words and a reading kind.** Proposed: @@ -559,7 +559,7 @@ All three were checked on a frozen, slotted throwaway dataclass on 2026-11-01. T | Sub-phase | What | Needs | |---|---|---| -| 4i-a | The pipeline reads the compiled sets through one internal object, not `Lexicon`'s public fields; the three tools move to it or to the index. No public change; a frozenset test stays a C-level attribute read, so the frame count holds (`_CALL_BASELINE`) | — | +| 4i-a | The pipeline reads the compiled sets through one internal object, not `Lexicon`'s public fields; the three tools move to it or to the index. No public change; a frozenset test stays a C-level attribute read, so the frame count holds (`_CALL_BASELINE`). Landed 2026-11-02 (decisions/vocabulary-model's 2026-11-02 #754, sub-phase 4i-a entry) | — | | 4i-b | The break: `Lexicon` holds readings, `add`/`remove` by kind, `readings_of` and `LexiconReading`, the narrowed constructor, the three refusals with their translation, the per-kind repr, the warning on a title reading made phrase-only by a `given` reading, the views reading the lexicon's kinds with rules/R8 and rules/R9 amended, the lazy base, the 2.x pickle translation; the tests and docs moved, `docs/migrate.rst`'s table, a release-log bullet | 4i-a | | 4i-c | The choice recorded on the `Lexicon`: chained choices union, a caller's readings applied after the choice, the strict mismatch raised, `|` refusing two choices, a pack's fragment joined under the caller's lexicon, the compile and its refusals at `parser_for`, the underscore choice parameters validated, `_apply_choice` retired; the chained-choices entry and 3j's dropped-listing clause superseded | 4i-b | | 4i-d | Optional: `Reading.languages` follows the choice | 4i-c | @@ -578,7 +578,7 @@ Each sub-phase is one PR, measured on its own base as phase 2's and phase 3's ar | 4f | `[rules]` compiled into a `Locale`'s policy patch; RU and TR_AZ rebuilt from their files. Landed (decisions/vocabulary-model's 2026-10-29 sub-phase 4f entry) | 3b | | 4g | the compound given name, a rule kind (#704): built 2026-10-30, `Policy.two_given_names` (rules/O7) | 4f | | 4h | the shape inventory per order (#469), starting with the title between a family and a given name under `FAMILY_FIRST`. Landed as an inventory, no reading moved: the arrangements each order is written in, listed with what each reads (decisions/vocabulary-model's 2026-10-30 #754, sub-phase 4h entry); the title between, and bare after, under a family-first order stays a name word, #938 declined on a survey of real names, which write the title in front (decisions/H5's 2026-10-30 #938 entry) | — | -| 4i | the `Lexicon` surface: configuring by reading, the compiled sets internal, and the choice recorded on the `Lexicon`. Decided 2026-11-02, to be built in four PRs (4i-a to 4i-d, "The `Lexicon` surface: configuring by reading" above) | 3b, 3j | +| 4i | the `Lexicon` surface: configuring by reading, the compiled sets internal, and the choice recorded on the `Lexicon`. Decided 2026-11-02, to be built in four PRs, 4i-a landed (4i-a to 4i-d, "The `Lexicon` surface: configuring by reading" above) | 3b, 3j | **Interleaved with phase 3** (its sub-phases 3a to 3j, above): 4a can start beside 3a, its `languages` waiting for 3a's index; 4d needs nothing of phase 3, and 4b and 4c need 3a's index only for a listed word's kind, a suffix word's and a particle's; they unblock its packs, `cs`'s generations (3f) and `pt-BR`'s agnomens (3h) on 4b, `es` and `pt` (3g) on 4d and 4g; 4f lands with or after 3b, since both decide what a shipped `Locale` holds; an `is` pack, which no phase 3 sub-phase lists, needs 3b, 4c and 4f. 3j, file-strict, needs 3b alone (3i, which name-strict would have read, is set aside), and 4e and 4h stand apart from both. So the order is 3a and 4a, then 4b to 4d with 3b and 3c (4b after 3a), then 4f, then the packs with 4g, with 4e at any point and 3j at any point after 3b. diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 0a3328d9..ee80297f 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -567,8 +567,68 @@ def _normpairs( return tuple(sorted(deduped.items())) +@dataclass(frozen=True, slots=True, eq=False, repr=False) +class _LexiconSets: + """The vocabulary sets a parse reads, held apart from the public + `Lexicon` fields: the pipeline, `_render`, `_parser` and the tools + read `lexicon._sets.titles`, never `lexicon.titles` (#754, 4i-a; + tests/v2/test_layering.py holds it). One attribute per + `_VOCAB_FIELDS` entry, in that order, each a frozenset, so a + membership test stays a C-level attribute read and costs no frame. + + Built once per `Lexicon`, in its `__post_init__`, from the + lexicon's own fields, and never compared, hashed or pickled: a + `Lexicon`'s equality and pickle are its fields', and a load or a + `dataclasses.replace` builds this again. 4i-b, which makes the + compiled fields internal, changes only how it is filled: from the + lexicon's readings rather than from its public fields, the default + from nameparser/config/_compiled.py as today, so the reading index + stays unloaded on `parse("John Smith")`; the derived markers the + views will read (a listed suffix word's kind, the patronymic + particles) join it as attributes of their own. Nothing that reads + it moves then. + + Not a mapping and no repr of its own: it holds whole word lists, + and a repr of them would be unbounded.""" + + titles: frozenset[str] + given_name_titles: frozenset[str] + suffix_acronyms: frozenset[str] + suffix_words: frozenset[str] + suffix_acronyms_ambiguous: frozenset[str] + particles: frozenset[str] + particles_ambiguous: frozenset[str] + conjunctions: frozenset[str] + conjunctions_ambiguous: frozenset[str] + bound_given_names: frozenset[str] + given_name_abbreviations: frozenset[str] + maiden_markers: frozenset[str] + surnames: frozenset[str] + honorific_tails: frozenset[str] + after_given_titles: frozenset[str] + suffix_words_ambiguous: frozenset[str] + agnomens: frozenset[str] + titles_ambiguous: frozenset[str] + title_modifiers: frozenset[str] + title_styles: frozenset[str] + title_phrases: frozenset[str] + given_names: frozenset[str] + + +class _HoldsSets: + """`Lexicon`'s one slot outside its dataclass fields, holding its + `_LexiconSets`. A slot inherited from a plain base, because a field, + even one built `init=False`, would change `dataclasses.fields`, the + pickled state and the 2.8 pickles' layout check, and 4i-a changes + nothing public; `@dataclass(slots=True)` takes no slot of its own + beside its fields. Set once, by `Lexicon.__post_init__`.""" + + __slots__ = ("_sets",) + _sets: _LexiconSets + + @dataclass(frozen=True, slots=True) -class Lexicon: +class Lexicon(_HoldsSets): """The vocabulary a parser matches against: which words are titles, particles, suffixes, and so on. Immutable and hashable. Start from :meth:`default` (the shipped vocabulary) or @@ -884,6 +944,9 @@ def __post_init__(self) -> None: f"suffix_words_ambiguous exists to impose; drop it from " f"suffix_acronyms or from suffix_words_ambiguous" ) + # last, from the normalized fields: what the parse reads + object.__setattr__(self, "_sets", _LexiconSets( + **{name: getattr(self, name) for name in _VOCAB_FIELDS})) # -- constructors ---------------------------------------------------- diff --git a/nameparser/_parser.py b/nameparser/_parser.py index c644f8e1..ea8bfd65 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -98,7 +98,7 @@ def __post_init__(self) -> None: script.value for script in self.policy.segment_scripts if not any(_SCRIPT_MATCHERS[script](entry) - for entry in self.lexicon.surnames)) + for entry in self.lexicon._sets.surnames)) if uncovered: names = ", ".join(uncovered) one = len(uncovered) == 1 @@ -329,14 +329,14 @@ def _apply_choice(lexicon: Lexicon, languages: frozenset[str], fields, (unmasked, masks) = _choice_delta( _reading_index_selected(languages, strict), strict) updates: dict[str, object] = { - name: (getattr(lexicon, name) - gone) | joined + name: (getattr(lexicon._sets, name) - gone) | joined for name, (gone, joined) in fields.items()} # a word the choice drops from a base set leaves that set's markers # too, a caller's own mark of it included: marking a word the # vocabulary no longer holds is the orphan Lexicon refuses for marker, base, _ in _SUBSET_FIELDS: if base in fields and fields[base][0]: - kept = updates.get(marker, getattr(lexicon, marker)) + kept = updates.get(marker, getattr(lexicon._sets, marker)) updates[marker] = kept - fields[base][0] # type: ignore[operator] if unmasked or masks: cap = {k: v for k, v in lexicon.capitalization_exceptions diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py index df1a4abd..b340107b 100644 --- a/nameparser/_pipeline/_assemble.py +++ b/nameparser/_pipeline/_assemble.py @@ -178,8 +178,9 @@ def _listed_whole(text: str, lexicon: Lexicon) -> bool: only of a word whose whole form no file holds and whose parts some file does (`_part_languages`), so other words pay no frame.""" n, bare = _forms(text) + sets = lexicon._sets for name in _VOCAB_FIELDS: - field = getattr(lexicon, name) + field = getattr(sets, name) if n in field or bare in field: return True return False @@ -367,7 +368,7 @@ def _listed_suffix(text: str, lexicon: Lexicon) -> bool: always tagged, bracketed or not. Asked only of a suffix word with no suffix tag, so other names pay nothing.""" n = _normalize(text).replace(".", "") - return n in lexicon.suffix_acronyms or n in lexicon.suffix_acronyms_ambiguous + return n in lexicon._sets.suffix_acronyms or n in lexicon._sets.suffix_acronyms_ambiguous # rules/A2: "a name with no name content parses to the empty @@ -377,7 +378,7 @@ def _listed_suffix(text: str, lexicon: Lexicon) -> bool: def assemble(state: ParseState) -> ParsedName: dropped = set(state.dropped) final: dict[int, Token] = {} - surnames = state.lexicon.surnames + surnames = state.lexicon._sets.surnames folds_middle = state.policy.middle_as_family tokens = state.tokens last = len(tokens) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 4ae9d7a3..b48c2d4b 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -554,7 +554,7 @@ def as_a_name(i: int) -> PendingReading: and _run_addresses_by_given( (tokens[j].text for k in range(name_pieces[0]) for j in pieces[k]), - state.lexicon.given_name_titles)) + state.lexicon._sets.given_name_titles)) return _name_alt(tokens[i], _slot_field(state, seg_idx, len(name_pieces), len(name_pieces) + 1, order), diff --git a/nameparser/_pipeline/_classify.py b/nameparser/_pipeline/_classify.py index 71b47c6e..9a80b6bb 100644 --- a/nameparser/_pipeline/_classify.py +++ b/nameparser/_pipeline/_classify.py @@ -140,31 +140,31 @@ def _tags_for(token: WorkToken, n: str, state: ParseState, tags = set(token.tags) if marker_tag is not None: tags.add(marker_tag) - if n in lex.titles: + if n in lex._sets.titles: tags.add("vocab:title") - if n in lex.given_name_titles: + if n in lex._sets.given_name_titles: tags.add("vocab:given-title") # rules/H8: "the word joins the title field" -- tagged only where the # word is a title too, so an entry outside `titles` is inert - if n in lex.after_given_titles and n in lex.titles: + if n in lex._sets.after_given_titles and n in lex._sets.titles: tags.add("vocab:after-given-title") if suffix_as_written(n, token.text, lex): tags.add("vocab:suffix") - if n in lex.suffix_words_ambiguous: + if n in lex._sets.suffix_words_ambiguous: # rules/S11: "it reads as the suffix only where the name has words # to spare" -- the ambiguous class's tag, so every site # that weighs a member weighs this one, and the cue tags that # tell `_pieces.listed_lean` how its writing leans tags.add(AMBIGUOUS_ACRONYM_TAG) tags.add(AMBIGUOUS_WORD_TAG) - if n in lex.agnomens: + if n in lex._sets.agnomens: tags.add(AGNOMEN_TAG) # rules/R8: "The kind of a word listed as a suffix word is the # shipped vocabulary's" -- a gated credential (`kc`, #965) reads # as a credential, not the generation the class otherwise holds elif listed_word_kind_tag(n) == CREDENTIAL_WORD_TAG: tags.add(CREDENTIAL_WORD_TAG) - elif n in lex.suffix_words: + elif n in lex._sets.suffix_words: tags.add("vocab:suffix-word") # rules/R8: "The kind of a word listed as a suffix word is the # shipped vocabulary's" -- read here, from the word as this @@ -172,14 +172,14 @@ def _tags_for(token: WorkToken, n: str, state: ParseState, kind_tag = listed_word_kind_tag(n) if kind_tag is not None: tags.add(kind_tag) - if n in lex.suffix_acronyms_ambiguous: + if n in lex._sets.suffix_acronyms_ambiguous: tags.add(AMBIGUOUS_ACRONYM_TAG) # rules/P7: "A one-letter word written with its period is an # initial and not a particle" -- both particle tags, so 'Ó.' is # neither half of the vocabulary - if n in lex.particles and not _spells_an_initial(n, token.text): + if n in lex._sets.particles and not _spells_an_initial(n, token.text): tags.add("particle") - if n in lex.particles_ambiguous: + if n in lex._sets.particles_ambiguous: tags.add("vocab:particle-ambiguous") # rules/R9: "a particle the shipped vocabulary holds as a # patronymic" -- its kind, read here as a suffix word's is @@ -195,7 +195,7 @@ def _tags_for(token: WorkToken, n: str, state: ParseState, # one letter as the writer sees it: a decomposed 'й' is two code # points (#665), so non-ASCII text is counted composed, after the # C-level tests have declined everything else - single_letter_connective = (n in lex.conjunctions + single_letter_connective = (n in lex._sets.conjunctions and token.text.upper() != token.text.lower() and (len(token.text) == 1 or not token.text.isascii() @@ -208,7 +208,7 @@ def _tags_for(token: WorkToken, n: str, state: ParseState, # emit where the branch is taken, not where an ambiguous tag # sits, and a vocab: tag records MEMBERSHIP, not the branch # taken, so it is the wrong shape of record here. - if n in lex.conjunctions_ambiguous: + if n in lex._sets.conjunctions_ambiguous: tags.add("initial") else: # a bare capital y joins here, where mixed case vetoes it @@ -236,17 +236,17 @@ def _tags_for(token: WorkToken, n: str, state: ParseState, initial = is_initial(token.text) or ( letter_one_case and len(token.text) == 1 and _INITIAL.fullmatch(token.text.upper()) is not None) - if n in lex.conjunctions and not initial: + if n in lex._sets.conjunctions and not initial: tags.add("conjunction") if initial: tags.add("initial") - if n in lex.bound_given_names: + if n in lex._sets.bound_given_names: tags.add("vocab:bound-given") # rules/H2: "an abbreviation the vocabulary reads as a given name # written short is that given name" -- the reading H2's shape test # weighs (`_pieces.is_leading_title`); a membership, so any spelling # carries it and only that test acts on it - if n in lex.given_name_abbreviations: + if n in lex._sets.given_name_abbreviations: tags.add(GIVEN_ABBREVIATION_TAG) # maiden markers are NOT tagged here: an entry may be a phrase whose # words are not markers on their own, and this function sees one @@ -289,7 +289,7 @@ def _tags_for(token: WorkToken, n: str, state: ParseState, elif derived == "suffix": tags.add("vocab:suffix") elif (derived == "shape" and token.role is None - and n not in lex.suffix_acronyms_ambiguous): + and n not in lex._sets.suffix_acronyms_ambiguous): # This branch and `_vocab.ambiguous_class_candidate` ask # the SAME question twice, of necessity -- `segment` runs # before `classify` and has no tags to read yet -- kept @@ -326,7 +326,7 @@ def _tags_for(token: WorkToken, n: str, state: ParseState, # what the predicate would decline anyway, asked first in # C: a listed member ('John Smith, MA'), and a word that # is not alphabetic capitals ('John Smith, Ph. D.') - and n not in lex.suffix_acronyms_ambiguous + and n not in lex._sets.suffix_acronyms_ambiguous and (token.text.isupper() and (token.text.isalpha() or "-" in token.text or "/" in token.text) @@ -376,7 +376,7 @@ def classify(state: ParseState) -> ParseState: # _normalize(text), passed in so callers normalize once"). folded = [_normalize(t.text) for t in state.tokens] marker_tags = tag_marker_runs(state.tokens, state.comma_offsets, - state.lexicon.maiden_markers, folded) + state.lexicon._sets.maiden_markers, folded) # rules/P3 says a maiden marker, taken as one, and the words it # takes, are not among the name's own words -- so the span and its # clause cut are _pieces.own_words', shared with the site that @@ -386,7 +386,7 @@ def classify(state: ParseState) -> ParseState: # walking the texts again, and classify's answer is the one it was # before the helper existed. own, clause_at = own_words(state.tokens, state.comma_offsets, - state.lexicon.maiden_markers, marker_tags) + state.lexicon._sets.maiden_markers, marker_tags) # ONE fact per parse, and it is recorded now (ParseState.one_case): # segment writes it first where C2's flag on a tail part asks, and # a fact two stages decide apart is what recording it prevents @@ -455,14 +455,14 @@ def classify(state: ParseState) -> ParseState: # a contested word keeps TITLE_AMBIGUOUS_TAG either way, for the # report where it opens the name (assign) lex = state.lexicon - if not (lex.titles_ambiguous.isdisjoint(folded) - and lex.title_modifiers.isdisjoint(folded)): + if not (lex._sets.titles_ambiguous.isdisjoint(folded) + and lex._sets.title_modifiers.isdisjoint(folded)): written = list(tokens) for at, seg in enumerate(state.segments): own_part = [i for i in seg if written[i].role is None] norms = [folded[i] for i in own_part] - if (lex.titles_ambiguous.isdisjoint(norms) - and lex.title_modifiers.isdisjoint(norms)): + if (lex._sets.titles_ambiguous.isdisjoint(norms) + and lex._sets.title_modifiers.isdisjoint(norms)): continue readings = title_readings( [readers[i].text for i in own_part], norms, lex, @@ -471,9 +471,9 @@ def classify(state: ParseState) -> ParseState: for i, title in zip(own_part, readings): tags = written[i].tags n = folded[i] - if n not in lex.titles_ambiguous and n not in lex.title_modifiers: + if n not in lex._sets.titles_ambiguous and n not in lex._sets.title_modifiers: continue - if n in lex.titles_ambiguous and n not in lex.titles: + if n in lex._sets.titles_ambiguous and n not in lex._sets.titles: tags = tags | {TITLE_AMBIGUOUS_TAG} if title: # in a title phrase: a title (its given-name title @@ -492,8 +492,8 @@ def classify(state: ParseState) -> ParseState: # of words pays no frame; a token carrying a role is delimited # content, which no pair reaches. split: dict[int, str] | None = None - acronyms = state.lexicon.suffix_acronyms - words = state.lexicon.suffix_words + acronyms = state.lexicon._sets.suffix_acronyms + words = state.lexicon._sets.suffix_words for seg in state.segments: for j in range(len(seg) - 1): a, b = tokens[seg[j]], tokens[seg[j + 1]] @@ -578,8 +578,8 @@ def classify(state: ParseState) -> ParseState: # token -- 'e and e e' emits twice; assemble withdraws the # report on a letter no name word follows (rules/P8, #721). if (one_case and "initial" in token.tags - and folded[i] in state.lexicon.conjunctions_ambiguous - and folded[i] in state.lexicon.conjunctions + and folded[i] in state.lexicon._sets.conjunctions_ambiguous + and folded[i] in state.lexicon._sets.conjunctions and i < clause_at and token.role is None and (len(token.text) == 1 or not token.text.isascii() diff --git a/nameparser/_pipeline/_comma.py b/nameparser/_pipeline/_comma.py index 5ec5a92b..6a0663d0 100644 --- a/nameparser/_pipeline/_comma.py +++ b/nameparser/_pipeline/_comma.py @@ -162,7 +162,7 @@ def _name_contrast(state: ParseState, seg0: Sequence[int], return False if clause_at is None: _, clause_at = own_words(state.tokens, state.comma_offsets, - state.lexicon.maiden_markers) + state.lexicon._sets.maiden_markers) lex = state.lexicon return any( i < clause_at and (tok := state.tokens[i]).role is None @@ -256,7 +256,7 @@ def _behind_a_generation(state: ParseState, seg0: Sequence[int], # is, and a maiden clause is no word of the name ('Jane nee Smith # Jr., AD' reads as 'Jane Jr., AD'; own_words, as for the contrast) _, clause_at = own_words(state.tokens, state.comma_offsets, - lex.maiden_markers) + lex._sets.maiden_markers) front = [i for i in seg0[:-1] if i < clause_at and tokens[i].role is None] if not _whole_name(front, tokens): diff --git a/nameparser/_pipeline/_extract.py b/nameparser/_pipeline/_extract.py index 2dfb7c49..b7784669 100644 --- a/nameparser/_pipeline/_extract.py +++ b/nameparser/_pipeline/_extract.py @@ -54,10 +54,10 @@ def _suffix_shaped(content: str, lexicon: Lexicon) -> bool: veto -- v1 deliberately skipped it here.""" stripped = _normalize(content) acronym = stripped.replace(".", "") - return ((stripped in lexicon.suffix_words - and stripped not in lexicon.suffix_words_ambiguous) - or (acronym in lexicon.suffix_acronyms - and acronym not in lexicon.suffix_acronyms_ambiguous) + return ((stripped in lexicon._sets.suffix_words + and stripped not in lexicon._sets.suffix_words_ambiguous) + or (acronym in lexicon._sets.suffix_acronyms + and acronym not in lexicon._sets.suffix_acronyms_ambiguous) or content.endswith(".")) @@ -94,7 +94,7 @@ def _maiden_marked(content: str, lexicon: Lexicon) -> bool: # test did for free. if len(words) < 2: return False - run = maiden_marker_run(words, lexicon.maiden_markers) + run = maiden_marker_run(words, lexicon._sets.maiden_markers) return (run > 0 and len(words) > run and not comma_follows_marker(words[run])) diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index fe73781e..2150e83d 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -1401,7 +1401,7 @@ def group(state: ParseState) -> ParseState: part_pieces, part_ptags, taken = _group_segment( part, additional, tokens, bound_join, None if family_comma else ambiguities, - state.lexicon.given_name_titles, + state.lexicon._sets.given_name_titles, opens_the_name=(seg_idx == 0 and not family_comma), one_case=state.one_case, site=site, reads=seg_reads) diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index 6ec893a6..d247b8aa 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -498,7 +498,7 @@ def _settle_delimited_members(state: ParseState, tokens: list[WorkToken], # escape dotted = ("." in token.text.strip(".") and _normalize(token.text).replace(".", "") - in lex.suffix_acronyms_ambiguous) + in lex._sets.suffix_acronyms_ambiguous) # after the name: behind its last name word, and a name word # there is, so a clause in a name of no word of its own # ('Dr. (MA)') is not settled @@ -559,7 +559,7 @@ def post_rules(state: ParseState) -> ParseState: and not _run_addresses_by_given( (tokens[i].text for i in _addressing_run(titles, givens[0])), - state.lexicon.given_name_titles)): + state.lexicon._sets.given_name_titles)): for i in givens: _retag(tokens, i, Role.FAMILY) # every rule below reads these lists; recompute after any @@ -582,7 +582,7 @@ def post_rules(state: ParseState) -> ParseState: and _run_addresses_by_given( (tokens[i].text for i in _addressing_run(titles, families[0])), - state.lexicon.given_name_titles)): + state.lexicon._sets.given_name_titles)): for i in families: _retag(tokens, i, Role.GIVEN) # recomputed for H1's reason, stated above diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 6852aba4..b2101725 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -348,7 +348,7 @@ def _one_case(state: ParseState) -> bool: if state.one_case is not None: return state.one_case return is_one_case(own_words(state.tokens, state.comma_offsets, - state.lexicon.maiden_markers)[0]) + state.lexicon._sets.maiden_markers)[0]) def _postnominal_behind_a_whole_name(state: ParseState) -> bool: @@ -446,7 +446,7 @@ def _peel_honorific_tail(state: ParseState, matching surname is a competing READING of the name, which a caller may prefer, while a shorter tail leaves a remainder that is not a name at all (김선생), so there is nothing to adjudicate.""" - tails = state.lexicon.honorific_tails + tails = state.lexicon._sets.honorific_tails if not tails: return state # The NAME's runs: more than segments[0], never every segment. A @@ -550,7 +550,7 @@ def _split_candidate(state: ParseState) -> int | None: # role is the order resolution's call, stop or no stop. Asked here, # with the core the split reads (#323), so the comma gate in front # learns it before asking the comma's decision (#613's /simplify) - if state.tokens[i].text.rstrip(FULL_STOPS) in state.lexicon.surnames: + if state.tokens[i].text.rstrip(FULL_STOPS) in state.lexicon._sets.surnames: return None return i @@ -574,7 +574,7 @@ def _split_surname_site(state: ParseState) -> ParseState: return state token = state.tokens[i] text = token.text - surnames = state.lexicon.surnames + surnames = state.lexicon._sets.surnames # The vocabulary match reads the token's CORE (#323). Without it # the stop WAS the remainder: '김.' matched its own head and became # 김 + '.'; the stop rides with the remainder instead, so '김민준.' diff --git a/nameparser/_pipeline/_segment.py b/nameparser/_pipeline/_segment.py index dadaaf48..f10ecaf6 100644 --- a/nameparser/_pipeline/_segment.py +++ b/nameparser/_pipeline/_segment.py @@ -86,7 +86,7 @@ def case_class() -> bool: if one_case is None: one_case = is_one_case(own_words( state.tokens, state.comma_offsets, - state.lexicon.maiden_markers)[0]) + state.lexicon._sets.maiden_markers)[0]) return one_case # The texts are built inline rather than through a helper @@ -155,7 +155,7 @@ def titles_and_suffixes(seg: tuple[int, ...]) -> bool: # rules/C10: "the word keeps the delimiter in its text and reads # as it would without it" -- a title with one glued ('Dr;') # reads as the title, as classify tags it (decisions/C10) - titles = state.lexicon.titles + titles = state.lexicon._sets.titles cores = glued_cores(state.policy.extra_suffix_delimiters) rest = tuple([i for i in seg if _normalize(t := state.tokens[i].text) not in titles diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 1a480e57..5d6b7029 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -249,7 +249,7 @@ def is_title_shaped(text: str, lexicon: Lexicon) -> bool: """ return (bool(_PERIOD_ABBREV.match(text)) and (text.isascii() or not in_initialless_script(text)) - and _normalize(text) not in lexicon.given_name_abbreviations) + and _normalize(text) not in lexicon._sets.given_name_abbreviations) def is_initial_shaped(text: str) -> bool: @@ -578,8 +578,8 @@ def lexicon_cue(n: str, lexicon: Lexicon) -> Cue: """The `ambiguous_lean` cue of the folded member `n`, read off the lexicon by a caller that has no tags to read (`is_wholly_suffix`); `_pieces.listed_lean` reads the same answer off classify's tags.""" - if n in lexicon.suffix_words_ambiguous: - return "none" if n in lexicon.agnomens else "word" + if n in lexicon._sets.suffix_words_ambiguous: + return "none" if n in lexicon._sets.agnomens else "word" return "acronym" @@ -698,22 +698,22 @@ def split_pair(first: str, second: str, lexicon: Lexicon) -> str | None: or in_initialless_script(first + second)): return None fold = _normalize(a + b) - if (fold in lexicon.suffix_acronyms_ambiguous - or not (fold in lexicon.suffix_acronyms - or fold in lexicon.suffix_words)): + if (fold in lexicon._sets.suffix_acronyms_ambiguous + or not (fold in lexicon._sets.suffix_acronyms + or fold in lexicon._sets.suffix_words)): return None na, nb = _normalize(a), _normalize(b) - if ((na in lexicon.suffix_acronyms or na in lexicon.suffix_words) - and (nb in lexicon.suffix_acronyms - or nb in lexicon.suffix_words)): + if ((na in lexicon._sets.suffix_acronyms or na in lexicon._sets.suffix_words) + and (nb in lexicon._sets.suffix_acronyms + or nb in lexicon._sets.suffix_words)): return None # a particle or a connective joins name words, which is what it is # doing beside one ('DE A JUNIOR', the A a surname written short); # written with its period a letter is an initial, not the connective # it spells ('SAM E.', rules/T5) for written, n in ((first, na), (second, nb)): - if not written.endswith(".") and (n in lexicon.particles - or n in lexicon.conjunctions): + if not written.endswith(".") and (n in lexicon._sets.particles + or n in lexicon._sets.conjunctions): return None return "split" @@ -735,12 +735,12 @@ def suffix_as_written(n: str, text: str, lexicon: Lexicon) -> bool: # removed periods only for the suffix_acronyms test); suffix WORDS # match on the plain normalized form a = n.replace(".", "") - if a in lexicon.suffix_acronyms_ambiguous and _dotted(text): + if a in lexicon._sets.suffix_acronyms_ambiguous and _dotted(text): return True - if ((a in lexicon.suffix_acronyms - and a not in lexicon.suffix_acronyms_ambiguous) - or (n in lexicon.suffix_words - and n not in lexicon.suffix_words_ambiguous)): + if ((a in lexicon._sets.suffix_acronyms + and a not in lexicon._sets.suffix_acronyms_ambiguous) + or (n in lexicon._sets.suffix_words + and n not in lexicon._sets.suffix_words_ambiguous)): return True # the joiners asked in C first, so a word holding neither pays no # frame @@ -806,7 +806,7 @@ def _is_suffix_strict_n(n: str, text: str, lexicon: Lexicon, and _INITIAL.fullmatch(text.upper()) is not None): # period-written ambiguous acronyms are exempt from the veto return _dotted(text) and \ - n.replace(".", "") in lexicon.suffix_acronyms_ambiguous + n.replace(".", "") in lexicon._sets.suffix_acronyms_ambiguous return suffix_as_written(n, text, lexicon) @@ -828,8 +828,8 @@ def is_suffix_lenient(text: str, lexicon: Lexicon) -> bool: bypassing the initial veto -- only safe in unambiguous positions (after a comma).""" n = _normalize(text) - return (n in lexicon.suffix_words - and n not in lexicon.suffix_words_ambiguous) \ + return (n in lexicon._sets.suffix_words + and n not in lexicon._sets.suffix_words_ambiguous) \ or _is_suffix_strict_n(n, text, lexicon) @@ -992,15 +992,15 @@ class by shape for the first time (measured regression, #516 if not _PERIOD_NOT_AT_END.match(text): return None chunks = [_normalize(c) for c in text.split(".") if c] - if any(c in lexicon.titles for c in chunks): + if any(c in lexicon._sets.titles for c in chunks): return "title" # rules/S11: "the word is part of that one word and is weighed only # by the shape the joined word wears" -- no chunk claim for a # suffix word that is also a name ('M.Filho') matched = [c for c in chunks - if c in lexicon.suffix_acronyms - or (c in lexicon.suffix_words - and c not in lexicon.suffix_words_ambiguous)] + if c in lexicon._sets.suffix_acronyms + or (c in lexicon._sets.suffix_words + and c not in lexicon._sets.suffix_words_ambiguous)] if matched and not all(len(c) == 1 and c.isascii() for c in matched): return "suffix" if (len(chunks) >= 2 and all(c.isalpha() for c in chunks) @@ -1055,8 +1055,8 @@ class at all). if "." in text: return False n = _normalize(text) - return (n in lexicon.suffix_acronyms_ambiguous - or n in lexicon.suffix_words_ambiguous) + return (n in lexicon._sets.suffix_acronyms_ambiguous + or n in lexicon._sets.suffix_words_ambiguous) # #516's all-caps half, ONE PREDICATE for the shape test and its @@ -1189,7 +1189,7 @@ def is_listed_given_name(text: str, lexicon: Lexicon) -> bool: `given_name_key`, so a caller's entry "José" matches as written and a list written without accents matches "José" too (rules/O6's veto, #959).""" - names = lexicon.given_names + names = lexicon._sets.given_names return bool(names) and (_normalize(text) in names or given_name_key(text) in names) @@ -1210,8 +1210,9 @@ def in_any_wordlist(n: str, lexicon: Lexicon) -> bool: the unlisted word it is ('John Smith, JAS' keeps suffix 'JAS'); and `given_names`, whose one reader is O6 (#959), for the same reason ('Ana Pérez, LUIS' reads as with the list empty).""" + sets = lexicon._sets for field in _LISTED_FIELDS: - if n in getattr(lexicon, field): + if n in getattr(sets, field): return True return False @@ -1276,8 +1277,9 @@ def claimed_as_non_name(n: str, lexicon: Lexicon) -> bool: asks it: such a word does not supply the name's case contrast. A different question from `in_any_wordlist`'s, which a surname list must also answer yes to.""" + sets = lexicon._sets for field in _NON_NAME_FIELDS: - if n in getattr(lexicon, field): + if n in getattr(sets, field): return True return False @@ -1350,7 +1352,7 @@ def ambiguous_class_candidate(text: str, lexicon: Lexicon, and period_joined_vocab(text, lexicon) == "shape"): return False n = _normalize(text) - return (n not in lexicon.suffix_acronyms_ambiguous + return (n not in lexicon._sets.suffix_acronyms_ambiguous and not suffix_as_written(n, text, lexicon)) return False @@ -1482,10 +1484,10 @@ def surname_unit_tags(text: str, lexicon: Lexicon, # for a token no whole-token title or suffix claimed ('JD.CPA'), # asked only of a token with a period, which it needs suffix = suffix_as_written(n, text, lexicon) or ( - "." in text and n not in lexicon.titles + "." in text and n not in lexicon._sets.titles and period_joined_vocab(text, lexicon) == "suffix") - particle = n in lexicon.particles and not ( - (leading and n in lexicon.titles) or _spells_an_initial(n, text)) + particle = n in lexicon._sets.particles and not ( + (leading and n in lexicon._sets.titles) or _spells_an_initial(n, text)) if suffix: return SURNAME_UNIT_TAGS if particle else _SUFFIX_ONLY return _PARTICLE_ONLY if particle else _NO_TAGS @@ -1508,7 +1510,7 @@ def in_a_title_phrase(text: str, lexicon: Lexicon) -> bool: """Whether a phrase of `lexicon.title_phrases` that classify can read holds `text`.""" return _normalize(text) in _phrase_words( - lexicon.title_phrases, lexicon.titles, lexicon.titles_ambiguous) + lexicon._sets.title_phrases, lexicon._sets.titles, lexicon._sets.titles_ambiguous) @functools.lru_cache(maxsize=16) @@ -1572,9 +1574,9 @@ def title_readings(texts: Sequence[str], folded: Sequence[str], reads as titles, where that is more than `titles` lists (classify's period-joined `Lt.Gov.`), else membership in `titles`. Only a word of the two sets is ever read out of the title.""" - contested = lexicon.titles_ambiguous - modifiers = lexicon.title_modifiers - titles = lexicon.titles + contested = lexicon._sets.titles_ambiguous + modifiers = lexicon._sets.title_modifiers + titles = lexicon._sets.titles # a word in titles is a title wherever it stands, whatever else it # is listed as; one of titles_ambiguous alone is listed as a title # to its neighbours and read by its phrase @@ -1582,7 +1584,7 @@ def title_readings(texts: Sequence[str], folded: Sequence[str], if titled is not None: reading = [r or t for r, t in zip(reading, titled)] listed = list(reading) - phrases = lexicon.title_phrases + phrases = lexicon._sets.title_phrases last = len(folded) - 1 for i, n in enumerate(folded): if n in titles or n not in contested or n in modifiers: @@ -1594,10 +1596,10 @@ def title_readings(texts: Sequence[str], folded: Sequence[str], # may be a given name and stands in no phrase is no partner # ('Grace Dean Smith') (prev is not None and reading[i - 1] - and (prev in modifiers or prev in lexicon.title_styles + and (prev in modifiers or prev in lexicon._sets.title_styles or f"{prev} {n}" in phrases)) or (nxt is not None - and ((nxt in _PHRASE_LINKS and nxt in lexicon.conjunctions) + and ((nxt in _PHRASE_LINKS and nxt in lexicon._sets.conjunctions) or (listed[i + 1] and f"{n} {nxt}" in phrases)))) # left to right, whether the run so far has passed 'of' or 'the': a # modifier after it is in the object of a phrase ('President of the @@ -1605,7 +1607,7 @@ def title_readings(texts: Sequence[str], folded: Sequence[str], # coordinating connective opens no object: 'Mr. and Mrs. Wing Lau' # is two salutations and a given name linked = [False] * len(folded) - conjunctions = lexicon.conjunctions + conjunctions = lexicon._sets.conjunctions for i in range(1, len(folded)): if reading[i - 1] or folded[i - 1] in conjunctions: linked[i] = linked[i - 1] or folded[i - 1] in _OBJECT_LINKS @@ -1679,13 +1681,13 @@ def name_word_count(texts: Sequence[str], lexicon: Lexicon, # rules/H9: "the word is a title only inside a title phrase" -- the # part before the comma is the head of the name, read as classify # reads it - titled = (None if lexicon.titles_ambiguous.isdisjoint(folded) - and lexicon.title_modifiers.isdisjoint(folded) + titled = (None if lexicon._sets.titles_ambiguous.isdisjoint(folded) + and lexicon._sets.title_modifiers.isdisjoint(folded) else title_readings(texts, folded, lexicon, head=True)) for i, n in enumerate(folded): text = texts[i] - title = n in lexicon.titles if titled is None else titled[i] - if n in lexicon.particles: + title = n in lexicon._sets.titles if titled is None else titled[i] + if n in lexicon._sets.particles: has_particle = True suffix = (is_suffix_lenient(text, lexicon) if lenient else is_suffix_strict(text, lexicon, letters_one_case)) diff --git a/nameparser/_render.py b/nameparser/_render.py index 8d586a13..7693b3bf 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -116,7 +116,7 @@ def _reads_as_conjunction(word: str, lex: Lexicon) -> bool: """ # the shape on the composed spelling, as _normalize composes the # lookup: a decomposed 'й.' is the initial its composed twin is (#542) - return bool(_normalize(word) in lex.conjunctions + return bool(_normalize(word) in lex._sets.conjunctions and not _INITIAL.fullmatch(unicodedata.normalize("NFC", word))) @@ -472,7 +472,7 @@ def _lowers_as_particle(normalized: str, word: str, role: Role, Shared with _cap_text, which asks it of each whole token before splitting the token into words, so a marked one ("d'") is asked as written rather than by its stem (#700).""" - return (normalized in lex.particles + return (normalized in lex._sets.particles and not _spells_an_initial(normalized, word) and role in (Role.MIDDLE, Role.FAMILY) and UNJOINED_TAG not in tags @@ -594,8 +594,8 @@ def _cap_word(word: str, role: Role, tags: frozenset[str], # agnomen takes no case cue and title-cases as any word does, and a # credential written as a word (`kc`, #965) carries a mask, which # this leaves to the exceptions below. - if (role is Role.SUFFIX and normalized in lex.suffix_words_ambiguous - and normalized not in lex.agnomens + if (role is Role.SUFFIX and normalized in lex._sets.suffix_words_ambiguous + and normalized not in lex._sets.agnomens and normalized not in lex.capitalization_exceptions_map): return word.lower() generation = role is Role.SUFFIX and "vocab:suffix" in tags @@ -635,7 +635,7 @@ def _cap_word(word: str, role: Role, tags: frozenset[str], # name word -- #459's given-role half, decided: repair follows the # role the parse chose ('qc mp' -> 'Qc MP'). if role is Role.SUFFIX and ( - undotted in lex.suffix_acronyms + undotted in lex._sets.suffix_acronyms or SHAPE_ACRONYM_TAG in tags): return word.upper() # rules/R4: "A roman numeral the parse put in the suffix role is @@ -741,7 +741,7 @@ def _joined_whole(text: str, role: Role, lex: Lexicon) -> str | None: masked = _apply_mask(text, mask) if masked is not None: return masked - if role is Role.SUFFIX and undotted in lex.suffix_acronyms: + if role is Role.SUFFIX and undotted in lex._sets.suffix_acronyms: return text.upper() return None @@ -809,7 +809,7 @@ def cap(word: str) -> str: if (first < at < last and not _DOTTED_INITIAL.fullmatch( unicodedata.normalize("NFC", part)) - and _normalize(part) in lex.conjunctions) + and _normalize(part) in lex._sets.conjunctions) else _sub_words(cap, part) for at, part in enumerate(parts)) @@ -843,7 +843,7 @@ def _split_pairs(tokens: tuple[Token, ...], repaired: tuple[Token, ...], fold = _normalize(joined.replace(".", "")) mask = lex.capitalization_exceptions_map.get(fold) cased = _apply_mask(joined, mask) if mask is not None else None - if cased is None and fold in lex.suffix_acronyms: + if cased is None and fold in lex._sets.suffix_acronyms: cased = joined.upper() # a case mapping that changes the length ('ß') has no place to # cut the pair back into its two words diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index a04fcf7b..09dcaeb3 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -349,9 +349,276 @@ def test_every_frozen_dataclass_carries_the_pickle_guards() -> None: continue if not dataclasses.is_dataclass(cls): continue + if cls.__name__ == "_LexiconSets": + # never pickled: a Lexicon pickles its fields and + # rebuilds this from them at load (#754, 4i-a) + continue if cls.__name__ == "Lexicon": assert "__getstate__" in cls.__dict__ assert "__setstate__" in cls.__dict__ continue assert cls.__dict__.get("__getstate__") is _guarded_getstate, cls assert cls.__dict__.get("__setstate__") is _guarded_setstate, cls + + +# -- the compiled sets are read through Lexicon._sets (#754, 4i-a) -------- + +_REPO = PKG.parent + +#: Where a parse reads vocabulary: every library module but `_lexicon`, +#: which owns the fields and fills `_sets` from them. A vocabulary +#: field read here must go through `._sets`, so that 4i-b can +#: make the compiled fields internal without touching a reader. +_LIBRARY_READERS = sorted( + p for p in [*PKG.glob("*.py"), *PKG.glob("_pipeline/*.py"), + *PKG.glob("locales/*.py")] + if p.name != "_lexicon.py") + +#: Every tool. Their own counters and records reuse the field names +#: (`local_corpora`'s `found.titles`), so a tool is checked by what a +#: read is taken OFF -- a lexicon, by its spelling or by a name bound +#: to one -- rather than by the library's rule that only `_sets` may +#: be read. +_TOOL_READERS = sorted((_REPO / "tools").rglob("*.py")) + +#: Names that hold no lexicon though they carry an attribute of a +#: vocabulary field's name: `read` is a `TailRead`, whose `titles` is +#: the title words of a segment's trailing run (_assign, _group). +_NOT_A_LEXICON = frozenset({"read"}) + + +def _lexiconish(node: ast.expr | None, aliases: frozenset[str]) -> bool: + """`lexicon`, `lex`, `.lexicon`, `Lexicon.default()` or + `Lexicon.empty()`, or a name bound to one of those (`aliases`).""" + if isinstance(node, ast.Name): + return node.id in ("lexicon", "lex") or node.id in aliases + if isinstance(node, ast.Attribute): + return node.attr == "lexicon" + return (isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr in ("default", "empty") + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "Lexicon") + + +def _from_sets(node: ast.expr | None) -> bool: + return isinstance(node, ast.Attribute) and node.attr == "_sets" + + +def _bindings(tree: ast.AST) -> list[tuple[str, int, ast.expr | None, + ast.expr | None]]: + """(name, line, value, annotation) for every binding of a plain + name: an assignment, annotated or augmented, a walrus, a for or + comprehension target, a parameter, a with or except target. The + value is None where the form gives a name no single value of its + own (a parameter, a loop target, one name of an unpacking).""" + out: list[tuple[str, int, ast.expr | None, ast.expr | None]] = [] + + def target(t: ast.expr, value: ast.expr | None) -> None: + if isinstance(t, ast.Name): + out.append((t.id, t.lineno, value, None)) + elif isinstance(t, (ast.Tuple, ast.List)): + for elt in t.elts: + target(elt, None) + elif isinstance(t, ast.Starred): + target(t.value, None) + + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + for t in node.targets: + target(t, node.value) + elif isinstance(node, (ast.AnnAssign, ast.AugAssign)): + if isinstance(node.target, ast.Name) and ( + isinstance(node, ast.AugAssign) or node.value is not None): + out.append((node.target.id, node.lineno, node.value, + getattr(node, "annotation", None))) + elif isinstance(node, ast.NamedExpr): + target(node.target, node.value) + elif isinstance(node, (ast.For, ast.AsyncFor, ast.comprehension)): + target(node.target, None) + elif isinstance(node, ast.withitem) and node.optional_vars: + target(node.optional_vars, None) + elif isinstance(node, ast.ExceptHandler) and node.name: + out.append((node.name, node.lineno, None, None)) + elif isinstance(node, ast.arguments): + for arg in [*node.posonlyargs, *node.args, *node.kwonlyargs, + *filter(None, (node.vararg, node.kwarg))]: + out.append((arg.arg, arg.lineno, None, arg.annotation)) + return out + + +def _lexicon_aliases( + bindings: list[tuple[str, int, ast.expr | None, ast.expr | None]], +) -> frozenset[str]: + """The names some binding gives a lexicon: by its value's spelling, + by another such name, or by a `Lexicon` annotation.""" + aliases: frozenset[str] = frozenset() + while True: + grown = aliases | { + name for name, _, value, annotation in bindings + if _lexiconish(value, aliases) or ( + annotation is not None + and "Lexicon" in ast.unparse(annotation))} + if grown == aliases: + return aliases + aliases = frozenset(grown) + + +def _direct_reads(source: str, library: bool) -> list[tuple[int, str]]: + """(line, field) for every vocabulary field `source` reads off a + lexicon rather than through its `_sets`: an attribute load; a + `getattr` of a lexicon, by its spelling or through a name bound to + one, or, in the library, of a field named as a constant off + anything but a `_sets`; and a name `sets` bound by any form but an + assignment from a `_sets` (which would make the attribute rule's + `sets` a lexicon), reported as `sets =`. + + Its limits, each a reach no reader has today: a lexicon handed + through a container, a call's return or an attribute other than + `.lexicon` (`self.lex.titles` in a tool) is not followed, and an + alias is per module, not per scope, so a name that holds a lexicon + in one function counts as one in every other.""" + from nameparser._lexicon import _VOCAB_FIELDS + tree = ast.parse(source) + bindings = _bindings(tree) + aliases = _lexicon_aliases(bindings) + found = [(line, "sets =") for name, line, value, _ in bindings + if name == "sets" and not _from_sets(value)] + sets_ok = not found + for node in ast.walk(tree): + if (isinstance(node, ast.Attribute) and node.attr in _VOCAB_FIELDS + and isinstance(node.ctx, ast.Load)): + base = node.value + if library: + if not (_from_sets(base) + or (isinstance(base, ast.Name) and ( + (base.id == "sets" and sets_ok) + or base.id in _NOT_A_LEXICON))): + found.append((node.lineno, node.attr)) + elif _lexiconish(base, aliases): + found.append((node.lineno, node.attr)) + elif (isinstance(node, ast.Call) and isinstance(node.func, ast.Name) + and node.func.id == "getattr" and node.args): + obj = node.args[0] + named = (len(node.args) > 1 + and isinstance(node.args[1], ast.Constant) + and node.args[1].value in _VOCAB_FIELDS) + if _lexiconish(obj, aliases) or ( + library and named and not _from_sets(obj) + and not (isinstance(obj, ast.Name) and obj.id == "sets" + and sets_ok)): + found.append((node.lineno, "getattr")) + return sorted(found) + + +def _readers() -> list[tuple[pathlib.Path, bool]]: + return ([(p, True) for p in _LIBRARY_READERS] + + [(p, False) for p in _TOOL_READERS]) + + +def test_the_reader_lists_name_files_that_exist() -> None: + # a renamed tool or stage would otherwise drop out of the check + # below in silence, an empty scope passing it + assert len(_LIBRARY_READERS) > 20 + for path, _ in _readers(): + assert path.is_file(), path + + +def _vocab_field_ids() -> list[str]: + from nameparser._lexicon import _VOCAB_FIELDS + return list(_VOCAB_FIELDS) + + +@pytest.mark.parametrize("field", _vocab_field_ids()) +def test_no_reader_takes_a_vocabulary_field_off_a_lexicon(field: str) -> None: + # 4i-a: the parse, the renderer, the parser and the tools read the + # vocabulary sets through Lexicon._sets (_lexicon._LexiconSets), + # never the public fields 4i-b makes internal. Per field, so that + # the field a stray read names is the failing test's id. + stray = [f"{path.relative_to(_REPO)}:{line}" + for path, library in _readers() + for line, name in _direct_reads(path.read_text(), library) + if name in (field, "getattr", "sets =")] + assert not stray, stray + + +@pytest.mark.parametrize("field", _vocab_field_ids()) +@pytest.mark.parametrize("spelling,library", [ + ("lexicon.{f}", True), ("state.lexicon.{f}", True), ("lex.{f}", True), + ("self.lexicon.{f}", True), ("Lexicon.default().{f}", True), + ("lexicon.{f}", False), ("lex.{f}", False), ("parser.lexicon.{f}", False), + ("Lexicon.default().{f}", False), +]) +def test_the_check_finds_a_direct_read(field: str, spelling: str, + library: bool) -> None: + # the negative control, per field and spelling: a read the check + # above must refuse + source = f"x = 'w' in {spelling.format(f=field)}\n" + assert _direct_reads(source, library) == [(1, field)] + through = f"x = 'w' in {spelling.format(f='_sets.' + field)}\n" + assert _direct_reads(through, library) == [] + + +@pytest.mark.parametrize("source,expected", [ + ("getattr(lexicon, name)\n", [(1, "getattr")]), + ("getattr(lexicon._sets, name)\n", []), + ("sets = lexicon\nsets.titles\n", [(1, "sets ="), (2, "titles")]), + ("sets = lexicon._sets\nsets.titles\n", []), + ("sets = lexicon._sets\ngetattr(sets, name)\n", []), + ("read.titles\n", []), + ("found.titles\n", [(1, "titles")]), + # getattr through an alias, or of a field named as a constant + ("lx = state.lexicon\ngetattr(lx, name)\n", [(2, "getattr")]), + ("def f(lx: Lexicon):\n getattr(lx, name)\n", [(2, "getattr")]), + ("getattr(obj, 'titles')\n", [(1, "getattr")]), + ("getattr(obj, name)\n", []), + # every form that binds `sets` but an assignment from a `_sets` + ("sets: object = lexicon\nsets.titles\n", [(1, "sets ="), (2, "titles")]), + ("(sets := lexicon)\nsets.titles\n", [(1, "sets ="), (2, "titles")]), + ("sets, other = lexicon._sets, 1\nsets.titles\n", + [(1, "sets ="), (2, "titles")]), + ("for sets in lexicons:\n sets.titles\n", [(1, "sets ="), (2, "titles")]), + ("def f(sets):\n return sets.titles\n", [(1, "sets ="), (2, "titles")]), + ("with open() as sets:\n sets.titles\n", [(1, "sets ="), (2, "titles")]), +]) +def test_the_check_finds_a_roster_read_and_an_alias( + source: str, expected: list[tuple[int, str]]) -> None: + assert _direct_reads(source, True) == expected + + +@pytest.mark.parametrize("source,expected", [ + # the AGENTS.md recipe's own spelling, and an alias of an alias + ("L = Parser().lexicon\nL.titles - L.suffix_words\n", + [(2, "suffix_words"), (2, "titles")]), + ("a = Lexicon.default()\nb = a\nb.particles\n", [(3, "particles")]), + ("def f(lx: Lexicon):\n return lx.titles\n", [(2, "titles")]), + ("def f(lx: 'Lexicon | None'):\n return lx.titles\n", [(2, "titles")]), + ("L = Parser().lexicon\ngetattr(L, name)\n", [(2, "getattr")]), + # what a tool's own records may spell + ("found.titles\nname.given_names\n", []), + ("getattr(found, 'titles')\n", []), +]) +def test_the_check_finds_a_tool_read_through_an_alias( + source: str, expected: list[tuple[int, str]]) -> None: + assert _direct_reads(source, False) == expected + + +#: The reads 4i-a moved, as the check counts them on today's sources +#: with `._sets` taken out again: the recorded negative control over +#: the real tree, so the check above cannot pass because its walk +#: reaches nothing. The library's 114 are 106 attribute loads, 5 roster +#: `getattr`s (3 of them through `sets`, which the undone `sets = +#: lexicon` makes a lexicon) and the 3 `sets =` bindings; the tools' +#: 16 are attribute loads, every tool scanned. +#: A reader added through `_sets` moves these, and they are re-taken +#: then: they count what the check would refuse with the move undone. +_UNGUARDED_READS = {"library": 114, "tools": 16} + + +def test_the_check_refuses_every_read_4i_a_moved() -> None: + counts = {"library": 0, "tools": 0} + for path, library in _readers(): + undone = path.read_text().replace("._sets", "") + counts["library" if library else "tools"] += len( + _direct_reads(undone, library)) + assert counts == _UNGUARDED_READS diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index e91a7680..5b071994 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -1240,3 +1240,16 @@ def test_a_shipped_phrase_entry_is_stored_as_written(field: str) -> None: f"it stores are not the ones the data module wrote -- " f"{sorted(frozenset(shipped) - getattr(Lexicon.default(), field))} " f"were folded away or changed") + + +def test_the_internal_sets_hold_each_field_under_its_own_name() -> None: + # 4i-a: the parse reads Lexicon._sets, built from the fields by + # keyword. The roster pinned here so that a field added to one and + # not the other fails at once (4i-b adds the derived markers); the + # identity check, so that a field filled from the wrong set does. + from nameparser._lexicon import _LexiconSets + assert [f.name for f in dataclasses.fields(_LexiconSets)] == list( + _VOCAB_FIELDS) + for lexicon in (Lexicon.default(), Lexicon.empty().add(titles={"x"})): + for name in _VOCAB_FIELDS: + assert getattr(lexicon._sets, name) is getattr(lexicon, name), name diff --git a/tools/differential/local_agreement.py b/tools/differential/local_agreement.py index f52bc7de..dd8e2817 100644 --- a/tools/differential/local_agreement.py +++ b/tools/differential/local_agreement.py @@ -162,7 +162,7 @@ def fold(text: str) -> str: def _title_words(lexicon: Lexicon) -> frozenset[str]: - return frozenset(lexicon.titles) | frozenset(lexicon.conjunctions) + return frozenset(lexicon._sets.titles) | frozenset(lexicon._sets.conjunctions) def unlisted(title: str, listed: frozenset[str]) -> bool: diff --git a/tools/es_given_names.py b/tools/es_given_names.py index 0c550a2c..43e04bd1 100644 --- a/tools/es_given_names.py +++ b/tools/es_given_names.py @@ -131,8 +131,8 @@ def excluded_words() -> set[str]: conjunction, a title or a suffix, folded as the list is.""" from nameparser import Lexicon lex = Lexicon.default() - pool = (lex.particles | lex.conjunctions | lex.titles - | lex.titles_ambiguous | lex.suffix_acronyms | lex.suffix_words) + pool = (lex._sets.particles | lex._sets.conjunctions | lex._sets.titles + | lex._sets.titles_ambiguous | lex._sets.suffix_acronyms | lex._sets.suffix_words) return {fold(w) for entry in pool for w in entry.split()} diff --git a/tools/name_usage.py b/tools/name_usage.py index a3f05fa9..7f364942 100644 --- a/tools/name_usage.py +++ b/tools/name_usage.py @@ -1937,15 +1937,15 @@ def _vocabulary(key: str) -> str: lexicon = Lexicon.default() low = key.lower() kinds = [] - if low in lexicon.suffix_acronyms_ambiguous: + if low in lexicon._sets.suffix_acronyms_ambiguous: kinds.append("ambiguous acronym") - elif low in lexicon.suffix_acronyms: + elif low in lexicon._sets.suffix_acronyms: kinds.append("acronym") - if low in lexicon.suffix_words: + if low in lexicon._sets.suffix_words: kinds.append("suffix word") - if low in lexicon.titles: + if low in lexicon._sets.titles: kinds.append("title") - if low in lexicon.particles: + if low in lexicon._sets.particles: kinds.append("particle") return ", ".join(kinds) or "-" @@ -1953,8 +1953,8 @@ def _vocabulary(key: str) -> str: def _listed_suffix(key: str) -> bool: from nameparser import Lexicon lexicon, low = Lexicon.default(), key.lower() - return (low in lexicon.suffix_acronyms_ambiguous - or low in lexicon.suffix_acronyms or low in lexicon.suffix_words) + return (low in lexicon._sets.suffix_acronyms_ambiguous + or low in lexicon._sets.suffix_acronyms or low in lexicon._sets.suffix_words) def _cell(usage: Usage, count: int | None, error: str) -> str: