From 23adec37616d0680766d84b47d150a1d64a44852 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 29 Oct 2026 13:45:30 -0700 Subject: [PATCH 1/7] Fold _opens_its_segment's unreachable fallback into one expression (#754, 4a) codecov/patch flagged the return for a given-part token in no segment, which the pipeline never builds; an empty default keeps parse total without a branch no input reaches. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 84ac249eaaa525fba4f0fe3a0b83dbaa8e2e9b7e) --- nameparser/_pipeline/_assemble.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py index 5254e7c42..ba1fa4727 100644 --- a/nameparser/_pipeline/_assemble.py +++ b/nameparser/_pipeline/_assemble.py @@ -103,12 +103,11 @@ def _opens_its_segment(state: ParseState, i: int) -> bool: word that is both a bound given name and a particle, read into the given part, so other names pay nothing.""" dropped = state.dropped - for seg in state.segments: - if i in seg: - return next((j for j in seg if j not in dropped - and state.tokens[j].role is not Role.TITLE), - None) == i - return False + # every given-part token stands in a segment; the empty default + # keeps parse total all the same + seg = next((s for s in state.segments if i in s), ()) + return next((j for j in seg if j not in dropped + and state.tokens[j].role is not Role.TITLE), None) == i def _listed_suffix(text: str, lexicon: Lexicon) -> bool: From 72499a45c0f8532e7dcbf713e009fedd6ebde9f8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 29 Oct 2026 14:11:06 -0700 Subject: [PATCH 2/7] Fill Reading.languages from the reading index (3.0, 4a's follow-up, #754) Each token's reading, and each report's alternative, now names the shipped language files that list the word the way it was read, looked up at assemble in 3a's reading index (#753) by the word and the kind taken. A reading by shape or position names none, and so does a word from a caller's own Lexicon or the Korean surname list. The field takes ADDED_IN and compare=False, so 2.7 and 2.8 pickles load with () and no comparison moves; no role, tag or alternative moves and the snapshot is unchanged. Two bounded memos keep the lookups off the frame count: steady per-parse frames are unchanged, and the first parse that meets a listed word pays about 1 ms once to load the index. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +- docs/design/decisions/A1-ambiguity-report.md | 2 +- docs/design/decisions/vocabulary-model.md | 12 +- docs/design/vocabulary-model-rfc.md | 10 +- docs/release_log.rst | 1 + docs/usage.rst | 23 +++- nameparser/_pipeline/AGENTS.md | 2 +- nameparser/_pipeline/_assemble.py | 114 ++++++++++++++++-- nameparser/_reading_index.py | 28 ++++- nameparser/_types.py | 28 ++++- .../v2/data/parsedname_v28_alternative.pickle | Bin 0 -> 685 bytes tests/v2/pipeline/test_assemble.py | 82 ++++++++++++- tests/v2/test_layering.py | 6 +- tests/v2/test_reprs.py | 8 +- tests/v2/test_types.py | 56 ++++++++- 15 files changed, 339 insertions(+), 37 deletions(-) create mode 100644 tests/v2/data/parsedname_v28_alternative.pickle diff --git a/AGENTS.md b/AGENTS.md index 55581b934..8bd66e907 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -205,7 +205,7 @@ Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely o ### Vocabulary (`nameparser/vocab/`) and the configuration layer (`nameparser/config/`) -**The default vocabulary is written per word, in TOML, and compiled.** `nameparser/vocab/.toml` holds one file per BCP 47 language tag, each `[[word]]` entry a `text` and its readings keyed by kind. What a file means, how an entry is built, the kinds and properties with the set each compiles into, exclusions, bearer counts and the steps for adding a word are in `nameparser/vocab/README.md`, the reference for anyone editing the TOML; `tests/v2/test_vocab.py` holds its kind, property, derived-marker and reserved-name tables to the names `build.py` accepts and the sets it compiles, so read it rather than restating it here. The parser never reads the TOML: `tools/vocab/build.py` runs the build checks that replaced the modules' import-time asserts (its docstring numbers them), turns every reading into an index row with its file's tag, and writes the rows as the committed reading index `nameparser/config/_index.py` (#753, phase 3a) and, compiled, as the committed `nameparser/config/_compiled.py`, frozenset literals one entry per line; the build never reads `_index.py` back. The compiler is `nameparser/_reading_index.py`'s `compile_readings`, with the kind and property tables it reads (`CATEGORY`, `BEHAVIORAL`, `GATED_WORDS`), so the build and phase 3's choice of languages compile alike; the build keeps the checks and loads that module by path, as it loads `_lexicon.py`, and `tests/v2/test_reading_index.py` holds that the index compiled with no choice is `_compiled.py` exactly. `readings_of(text)` there is the run-time reader of the index, every reading of a word with its file's tag, loaded on first call. Each `config/` module re-exports its constants from `_compiled.py` (`TITLES = _compiled.TITLES`) under its autodoc docstring, so the public names and the docs did not move. `tests/v2/test_vocab.py` fails when `_compiled.py` is stale, and `tests/v2/test_reading_index.py` when the index is, each naming the refresh command, and `test_vocab.py` gives every build check a mutation. The model, the phasing and what was declined: decisions/vocabulary-model. +**The default vocabulary is written per word, in TOML, and compiled.** `nameparser/vocab/.toml` holds one file per BCP 47 language tag, each `[[word]]` entry a `text` and its readings keyed by kind. What a file means, how an entry is built, the kinds and properties with the set each compiles into, exclusions, bearer counts and the steps for adding a word are in `nameparser/vocab/README.md`, the reference for anyone editing the TOML; `tests/v2/test_vocab.py` holds its kind, property, derived-marker and reserved-name tables to the names `build.py` accepts and the sets it compiles, so read it rather than restating it here. The parser never reads the TOML: `tools/vocab/build.py` runs the build checks that replaced the modules' import-time asserts (its docstring numbers them), turns every reading into an index row with its file's tag, and writes the rows as the committed reading index `nameparser/config/_index.py` (#753, phase 3a) and, compiled, as the committed `nameparser/config/_compiled.py`, frozenset literals one entry per line; the build never reads `_index.py` back. The compiler is `nameparser/_reading_index.py`'s `compile_readings`, with the kind and property tables it reads (`CATEGORY`, `BEHAVIORAL`, `GATED_WORDS`), so the build and phase 3's choice of languages compile alike; the build keeps the checks and loads that module by path, as it loads `_lexicon.py`, and `tests/v2/test_reading_index.py` holds that the index compiled with no choice is `_compiled.py` exactly. `readings_of(text)` there is the run-time reader of the index, every reading of a word with its file's tag, loaded on first call; `languages_of(text, categories)` is the reader a parse calls, from assemble, for `Reading.languages` (3.0), so the first parse that reads a listed word loads the index. Each `config/` module re-exports its constants from `_compiled.py` (`TITLES = _compiled.TITLES`) under its autodoc docstring, so the public names and the docs did not move. `tests/v2/test_vocab.py` fails when `_compiled.py` is stale, and `tests/v2/test_reading_index.py` when the index is, each naming the refresh command, and `test_vocab.py` gives every build check a mutation. The model, the phasing and what was declined: decisions/vocabulary-model. The `config/` SETS are frozen since 2.2 (#293): there is no `.add()`/`.remove()` on any of them, so a default word list is changed at runtime by configuring a `Lexicon`, never by editing the constant, and the shipped default is changed by editing the TOML and rebuilding. **Neither dict was frozen, and `CAPITALIZATION_EXCEPTIONS` is not covered by anything else either.** `REGEXES` is a compiled-pattern table rather than vocabulary and was never in #293's scope; `CAPITALIZATION_EXCEPTIONS` is vocabulary-shaped and is a decided, in-scope exemption. So the hazard the freeze closes is still live for it: `CAPITALIZATION_EXCEPTIONS['dclinpsy'] = 'DClinPsy'` after the first parse misses the cached `Lexicon.default()`. Same advice — configure the object (`dataclasses.replace(lexicon, capitalization_exceptions=...)`). `tests/v2/test_contracts.py::test_every_vocabulary_constant_is_frozen` names both dicts as explicit exemptions rather than letting its `isinstance` filter drop them. @@ -227,7 +227,7 @@ The `config/` SETS are frozen since 2.2 (#293): there is no `.add()`/`.remove()` The 2.0 rewrite landed as underscore-private modules beside the v1 code, which 2.0 and then 3.0 deleted. These conventions apply to all of it. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. - **Module layout**: every module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`, `_reading_index.py`, `_pipeline/`, `_parser.py`). The public import surface is exactly `nameparser`, `nameparser.locales` and the vocabulary data modules under `nameparser.config`; `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. -- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only, e.g. `config/maiden_markers.py`); `_reading_index` imports nothing internal at module level and the index, `config/_index.py`, at call time, since `tools/vocab/build.py` loads it by path; `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types`, `_lexicon` (for `Lexicon.default()` and `_normalize`) and `_policy` (for the scripts that write no initials); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers; `_parser` sits on everything except `_render`; `__main__` imports only the public package; locale pack modules (`locales/*.py`) import `_locale`/`_lexicon`/`_policy`/`_types` only (`_types` joined the list in #272 — `locales/ja.py`'s segmenter factory constructs a `Segmentation`), and the `locales/__init__` additionally lazy-imports its packs (PEP 562). Extend the test's `ALLOWED` table when adding a module. +- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only, e.g. `config/maiden_markers.py`); `_reading_index` imports nothing internal at module level and the index, `config/_index.py`, at call time, since `tools/vocab/build.py` loads it by path; `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types`, `_lexicon` (for `Lexicon.default()` and `_normalize`) and `_policy` (for the scripts that write no initials); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers, and `_assemble` also `_reading_index`, for `Reading.languages`; `_parser` sits on everything except `_render`; `__main__` imports only the public package; locale pack modules (`locales/*.py`) import `_locale`/`_lexicon`/`_policy`/`_types` only (`_types` joined the list in #272 — `locales/ja.py`'s segmenter factory constructs a `Segmentation`), and the `locales/__init__` additionally lazy-imports its packs (PEP 562). Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. `Role` is a `StrEnum`: members compare as their field-name strings, and `tokens_for()` coerces strings via `_coerce_enum`. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). **When the message hands the reader code to paste, that code has to survive a type checker, and then RUN** — nameparser ships `py.typed`, and a placeholder can type-check and still raise: the 2.4 release sweep's first fix to the `capitalization_exceptions` hint offered `('key', 'Value')`, which passes mypy and fails #459's mask check, so `test_add_capitalization_exceptions_raises_pointing_at_replace` now executes the offered code. #337's segmenterless warning offered `Policy(segment_scripts=())`, an `arg-type` error, because these fields are annotated with what they STORE rather than everything the constructor accepts. Prefer the `frozenset()` / `()` spellings in messages and docstrings, and pin the offered spelling in a test — the warning tests matched on `ja_segmenter` and never checked the actionable half of the message. **A warning emitted in `Parser.__post_init__` needs `parser_for` to re-emit it from its own frame** (the `catch_warnings(record=True)` block at its return): `__post_init__`'s `stacklevel` is sized for direct `Parser(...)` construction, and through `parser_for`'s extra frame the default one-line rendering attributes the warning to the library's own `return Parser(...)` — the exact call the message tells the user to change becomes invisible. No single stacklevel serves both entry points; a new construction warning gets the re-emission for free, but a new CONSTRUCTION SITE for `Parser` inside this package needs its own re-emission or its callers get library-attributed warnings (#337 review). diff --git a/docs/design/decisions/A1-ambiguity-report.md b/docs/design/decisions/A1-ambiguity-report.md index 50a22f79f..bd4b47541 100644 --- a/docs/design/decisions/A1-ambiguity-report.md +++ b/docs/design/decisions/A1-ambiguity-report.md @@ -35,7 +35,7 @@ Open: none in this file. DECIDED: four sources, where the RFC's "Readings on the parse" named three. `vocabulary` is a listed word in the reading's category, `declared` a word the vocabulary also declares a name (the derived markers of decisions/vocabulary-model's 2026-10-25 #752 entry, and a given name written short), `shape` a roman numeral, dotted letters or a one-letter initial, and `position` an ordinary word nothing lists, a name only by where it stands: `John Smith PhD Jones` gives `position` for `Jones` where `JOHN SMITH MA` gives `declared` for `MA`, which is the case the plan needed the source for. A shape-admitted member of the ambiguous class carries the class's tag beside the shape tag, so the shape tag decides that it declares no name. DECIDED: every emitter of every fork kind fills it at its decision site (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE). Two stages run before the field exists, classify's connective-or-initial letter and group's particle chain, and defer it (`_state.Deferred.JOINED` and `UNCHAINED`) for assign to name off the positions it reads: the connective's join takes in the name word on each side, so where those stand in two pieces the joined unit takes the place of the first (`JOSE E MARIA SANTOS` gives `given`), and where both are already inside the letter's own piece the join changes nothing the piece did not hold and the letter lands where its piece lands, named at assemble (`MARIA DA SILVA E SOUZA`, a case row, gives `family`); a join that would leave the name one word is none, rules/P3 keeping a single-letter connective in a three-word name a name word, so there too the letter keeps its own field (`JOHN E SMITH` gives `middle`), where a family comma's surname is the word that lets the given part join (`SMITH, JOHN E MARY` gives `given`); the unchained particle stands as one more unit in front of its piece. One assign cannot place stays unnamed and surfaces as no alternative, in two shapes on 2026-10-26: a connective letter opening its part, which has no word before it to join, so no field could hold the reading (`E SMITH JONES`, a case row since; `e j smith` and `e and e` in the committed corpora), and in the case sweep the chain's report in `Freiherr von Berg Dr. and Ed. Prof.`, under each order, whose chained piece ends in a title a connective's join made. The first was found by the design-docs review; rules/A1 names it. The three kinds that report a shape (comma-structure, unbalanced-delimiter, segmentation) carry none; `order` is reserved and emitted nowhere. ACCEPTED: a listed suffix word that is neither a generation by its tag nor an agnomen reads as `post-nominal`, so `jr` and `esq` alike, because telling them apart needs the per-word kind the reading index below would carry; only H4's input of nothing but post-nominals reaches it (`Rinpoche`). And the field holds every other word fixed, so it is not the field a re-parse with the word's reading removed would give wherever that re-parse reads something else differently: on 2026-10-26, 14 of the sweep's (row, order) pairs differ from a plain stand-in word that way. In 11 the stand-in moves a neighbour's reading (`John Smith VI Jones`, where a plain word for `VI` frees `Jones` from the run); in 3 it moves its own, `Andrew Perkins (X.Y.Z.)`, whose dotted content rules/S1 reads as if written bare, so its name reading is the bare word's, `family`, where a plain word in the brackets stays a nickname. A further 296 reports (299 once the PR review's `MARIA DA SILVA E SOUZA` row joined, 302 with #903's `Nasreen Sultana, MBBS`) cannot be checked that way at all, since the stand-in lands in the taken field: their decision was the position's. - DECLINED: the reading's `languages`, the files holding it. It needs a per-word reading index compiled into `_compiled.py`, the first parse-time use of the TOML's descriptive data. Measured 2026-10-26 on Python 3.11 on Derek's machine with bytecode cached, as an installed wheel has it: an index of the 1,790 words that hold a reading, 1,975 readings by word and kind, each with its files, added as a dict literal at the end of `_compiled.py`, costs about 0.35 ms of an 18.2 ms `import nameparser` (the median of 30 fresh processes, twice each, under a `PYTHONPYCACHEPREFIX`), about 2%, and a lookup costs each report a `_normalize` call. RECIPE for the index: for each `nameparser/vocab/*.toml` file and each `[[word]]` table, every table-valued key but `excluded` is a reading; group them by `text`, then kind, collecting the file stems. The cost is small; the reason is that the index describes the default vocabulary, not the `Lexicon` a parse used, so a word a caller or a pack adds would report no language and a word one removes would still report its file. It waits for 3.0 with `Token.reading`, which needs the same index. DECLINED too: naming a name reading's field by parsing again with the reading removed, which is exact and costs a parse per report, quadratic over the scaling shapes that report once per unit. + DECLINED (for 2.7; carried out in 3.0, decisions/vocabulary-model's 2026-10-29 #754 entry on `Reading.languages`): the reading's `languages`, the files holding it. It needs a per-word reading index compiled into `_compiled.py`, the first parse-time use of the TOML's descriptive data. Measured 2026-10-26 on Python 3.11 on Derek's machine with bytecode cached, as an installed wheel has it: an index of the 1,790 words that hold a reading, 1,975 readings by word and kind, each with its files, added as a dict literal at the end of `_compiled.py`, costs about 0.35 ms of an 18.2 ms `import nameparser` (the median of 30 fresh processes, twice each, under a `PYTHONPYCACHEPREFIX`), about 2%, and a lookup costs each report a `_normalize` call. RECIPE for the index: for each `nameparser/vocab/*.toml` file and each `[[word]]` table, every table-valued key but `excluded` is a reading; group them by `text`, then kind, collecting the file stems. The cost is small; the reason is that the index describes the default vocabulary, not the `Lexicon` a parse used, so a word a caller or a pack adds would report no language and a word one removes would still report its file. It waits for 3.0 with `Token.reading`, which needs the same index. DECLINED too: naming a name reading's field by parsing again with the reading removed, which is exact and costs a parse per report, quadratic over the scaling shapes that report once per unit. DECIDED: the pickle guard widened. `_types._guarded_setstate` refused any pickle whose field set differed from the class's, so the new field would have broken every `ParsedName` pickled with a report. A field added after its type shipped now carries `metadata={ADDED_IN: "2.7"}` and a default, and a pickle missing it loads with the default; a missing field without the marker (with a default or not: `Token.tags`, `ParsedName.ambiguities`), an unknown field and a renamed one are still refused. MEASURED 2026-10-26: each of the seven refusal rows in tests/v2/test_types.py raises under the pre-2.7 guard as under this one, and the admitted row and a `ParsedName` pickled by the 2.6.0 wheel (committed as tests/v2/data/parsedname_v26.pickle) raise under the pre-2.7 guard and load under this one; pickles from the 2.0.0 and 2.3.0 wheels load too. DECIDED: the snapshot records the alternatives, one string per report that carries one, in report order (`GIVEN_OR_FAMILY: name as family (position)`), so a later change moving one shows in the snapshot's diff, and losing one of two identical ones does too (`DOE, JOHN MA JD` holds two; the first draft recorded a set, which the PR review found hiding that). A snapshot's first line names the keys its recording tree could record (`snapshot.HEADER`), and a comparison leaves out a key either side did not record, read off that header rather than off whether a row holds one, which a 2.7 snapshot of names no report reaches would have failed; a snapshot without the header, a tag's or a local corpus's, records no alternatives (`snapshot.RECORDED_LATER`). `compare.py --baseline` against a release before 2.7 leaves them out too, its worker saying the wheel has no `Ambiguity.alternative`: on 2026-10-26 `--baseline 2.6.0` reported 1,088 entries moved with them compared, 867 of them only the key appearing, and 221 without, the role and ambiguity moves alone; `--since v2.6.0` reported 357 before and after. RECIPE for the measurements below: `uv run python tools/differential/snapshot.py`, `uv run python tools/differential/cldr_agreement.py`, and for each local corpus `snapshot.py --local --base ` and `local_agreement.py psc-individuals --base `. MISSED, then caught in review: the first draft named four fields wrong, each found by the code review's stand-in oracle on shapes no case row held. A credential or numeral with no name word before it counted the name word the carve-out makes as none (`PhD VI` gave `given`, where `family`). A numeral behind a particle P6 then attached took the particle's attached field (`Smith, John van VI` gave `family`, where `middle`), its test of P6's tag asked before P6 wrote it. A bound given name took a numeral behind it with no family word left for its join (`abdul VI` gave `given`, where `family`). And P6's own reports named the particle's field without asking whether the chain in front would take a name word in (`Juan de la Vega van` under FAMILY_FIRST gave `middle`, where `given`). The second review, of the fix commit, found two of those fixes narrowing elsewhere: the chain test the P6 reports now asked took a bound join for a chain (`Smith, Abu Mary van` gave `given`, where `middle`), and the bound join's licence was granted to a family comma alone, not to a given-name title (`Sir abdul VI` gave `family`, where `given`); it also found the P1 fold unmodelled (`Dr. mc VI` under FAMILY_FIRST gave `given`, where `family`). The PR's own review then found three more, and a review of its fixes a fourth. `capitalized()` rebuilt every report without its alternative, unseen by any test because the field is not compared, so the rebuilt name still equalled the original. A connective inside a joined piece got none, the join assumed to merge two pieces: 1,651 of the 4,967 connective reports in the tse-donors local corpus on 2026-10-26, 1 after the fix, the opening letter's. And the given-name title licence for the bound join was asked of every title token rather than of the leading run group's join asks through `_lexicon._run_addresses_by_given` (`abdul VI Sir.` gave `given`, where `family`; `Sir Dr. abdul VI` under FAMILY_FIRST, the run's last word not addressing by the given name, likewise). A review of those fixes found the connective's field ignoring P3's three-word carve-out, inherited from the first draft (`JOHN E SMITH` gave `given`, where its twin `John e Smith` reads middle 'e'; under FAMILY_FIRST `family`, where `given`). The shapes are rows of `_REVIEW_SHAPES` in tests/v2/test_cases.py, which fails 34 of its then 48 parses with the first round's fixes reverted, 14 with the second's, and 3 of its 57 with the third's licence fix; `test_an_alternative_has_its_recorded_kind_role_and_source` holds the reports the stand-in oracle cannot check (a credential run's, whose stand-in the same run reads, and a one-case connective's, whose stand-in is an initial too), one row per branch choosing a kind or a source, and the `middle_as_family` fold; and `test_capitalized_keeps_each_reports_alternative` compares the alternatives themselves. diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index 945535d14..c7622b78c 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -275,16 +275,22 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) - **2026-10-28 #754, the `Lexicon` surface (Derek) — A CALLER CONFIGURES VOCABULARY BY READING, AND THE COMPILED SETS BECOME INTERNAL.** Derek decided in chat on 2026-10-28, choosing it over keeping a public field per compiled set and over keeping 2.0.0's eleven while hiding the eleven added since. A caller states what a word may be in the language files' terms (a `given` reading on `guru`, no credential reading on `dc`) and `Lexicon` derives the sets and gates as the build does. The reasons: the set mechanics are hard to keep straight even for the maintainer, a reading is what a caller can know about a word, and it is the form a pack contribution takes. Today the same configuration needs rules/H9's set algebra, `Lexicon.default().remove(titles={'guru'}).add(titles_ambiguous={'guru'})` (`Guru Prasad Sharma` → given `Guru`) and `remove(suffix_acronyms={'dc'})` (`Ram Bahadur DC` → family `DC`), while `add(suffix_acronyms_ambiguous={'dc'})` leaves suffix `DC` (2026-10-28, run on this branch's base). A caller's own removal may settle a clash: decision 15 binds the default and the packs, not a caller's data. `dataclasses.fields(Lexicon)` gives 22 public fields on 2026-10-28, eleven since 2.0.0's eleven; the RFC's Phase 4 "Left open" holds the weighing. A break of the frozen `Lexicon`, recorded in decisions/3-0-reevaluations. Open: the method names, and whether a caller's own language file can join the pool (phase 3's item 8 defers third-party files). The 2.x warning the freeze asks for was waived on 2026-10-29, 2.8 being the last 2.x feature release (decisions/3-0-reevaluations, THE LEXICON WARNING). - **2026-10-29 #753, phase 3a — THE READING INDEX AND THE SHARED COMPILER LAND, AND A NAME READING MAY BE WRITTEN `default = false`; NO PARSE MOVES.** Carries out items 3 and 5 of the 2026-10-27 #753 entry, and 3a of its item 11. - - **What landed.** tools/vocab/build.py writes a second generated module, nameparser/config/_index.py, beside _compiled.py: for each word, one row per reading in every file, on by default or not, `(tag, kind, properties)`, the properties those the compiler reads (each category's behavioral ones, and `same_as` on a `given` reading); counts, notes, exclusions and the stated `may_be_given` stay out, being checked at build time and compiled to nothing. The compiler moved into nameparser/_reading_index.py (`compile_readings`, with `CATEGORY`, `BEHAVIORAL` and `GATED_WORDS`), which the build loads by path as it loads _lexicon.py; the build keeps the checks and compiles _compiled.py from the index rows, so the module the parser reads is the index compiled. tests/v2/test_reading_index.py holds the no-choice invariant: `compile_readings(index())`, rendered as the build renders it, is _compiled.py byte for byte. The run-time reader is `readings_of(text)`, every `IndexReading` (text, tag, kind, props) of a word, for `Reading.languages`, which 4a (#942) adds but leaves unfilled and a follow-up after 3a and 4a both merge fills; `IndexReading.kind` is the vocabulary's kind (`credential`, `preposition`), and mapping it to a `ReadingKind` is that follow-up's. + - **What landed.** tools/vocab/build.py writes a second generated module, nameparser/config/_index.py, beside _compiled.py: for each word, one row per reading in every file, on by default or not, `(tag, kind, properties)`, the properties those the compiler reads (each category's behavioral ones, and `same_as` on a `given` reading); counts, notes, exclusions and the stated `may_be_given` stay out, being checked at build time and compiled to nothing. The compiler moved into nameparser/_reading_index.py (`compile_readings`, with `CATEGORY`, `BEHAVIORAL` and `GATED_WORDS`), which the build loads by path as it loads _lexicon.py; the build keeps the checks and compiles _compiled.py from the index rows, so the module the parser reads is the index compiled. tests/v2/test_reading_index.py holds the no-choice invariant: `compile_readings(index())`, rendered as the build renders it, is _compiled.py byte for byte. The run-time reader is `readings_of(text)`, every `IndexReading` (text, tag, kind, props) of a word, for `Reading.languages`, which 4a (#942) left unfilled and its follow-up fills (the 2026-10-29 #754 entry on `Reading.languages` below, which reads the tags through `languages_of` beside it); `IndexReading.kind` is the vocabulary's kind (`credential`, `preposition`), and mapping it to a `ReadingKind` is that follow-up's. - **A name reading written `default = false` derives no marker in the default** (item 3, the half before a pack turns it on): the word's other readings compile as they would without it. So it counts for none of the checks guarding the default's compile: an agnomen on by default needs a name reading on by default (check 11, else it reads ungated), and a bound given name that is also a particle, which needs a `given` reading on by default (else Lexicon raises at import), is refused by check 11's particle half below before check 9 reads it. A name reading off beside a suffix word kind no gate reads is still refused, gating nothing when on either. - **Left open: a particle whose word's only `given` readings are off is refused.** Check 11 asks such a particle to state `may_be_given = false` exactly where its word has no `given` reading. The default compiles it never-given, a pack turning the given reading on compiles it possibly given, and which of the two the stated property describes is a choice no 3.0 pack needs yet (`nl`'s prepositions wait with strict). The build refuses the shape, naming both ways out, rather than pick a side; the first pack that needs it decides, with its measurement. - **The index is written grouped by word, and an `IndexReading` is built only for a word looked up.** On a flat draft of this change, one row per line, grouping the rows at run time took about 0.56 ms; building every row's object takes about 1.4 ms (`index()`, which only a compile over the whole pool calls); each against the import's own cost, so the build writes one line per word and the lookup table is a single `dict()` call. A change to a word's readings is still a one-line diff. - - **The index loads on the first `readings_of` call, not inside `Lexicon.default()` (decided 2026-10-29, Derek).** SUPERSEDES: item 5's "loaded for every caller with `Lexicon.default()`". Proposed by 3a and chosen over loading it with the default lexicon. What it buys depends on the state of `Reading.languages`. Today, and until the follow-up that fills it after 3a and 4a (#942) both merge, no parse reads the index, so no caller pays the import, parsing or not, where loading it with the default lexicon would cost every parsing caller the import for nothing. Once the follow-up fills `Reading.languages`, the first parse will load it, so every parsing caller will pay it once, as item 5 measured, and only a caller who imports nameparser without parsing is spared. Wherever the follow-up calls `readings_of`, the import moves with the call. + - **The index loads on the first `readings_of` call, not inside `Lexicon.default()` (decided 2026-10-29, Derek).** SUPERSEDES: item 5's "loaded for every caller with `Lexicon.default()`". Proposed by 3a and chosen over loading it with the default lexicon. What it buys depends on the state of `Reading.languages`. Until the follow-up that fills it (the 2026-10-29 #754 entry on `Reading.languages` below), no parse read the index, so no caller paid the import, parsing or not, where loading it with the default lexicon would have cost every parsing caller the import for nothing. Since that follow-up, the first parse to read a word from the vocabulary loads it, so every caller who parses such a name pays it once, about 1 ms on that parse (measured there), and a caller who imports nameparser without parsing, or parses only words no file lists, is spared; the import moved with the call, into assemble. - **Measured** 2026-10-29 on `9210f221` plus this change, Python 3.11.17, medians of fresh processes with bytecode cached as an installed wheel has it: importing the index takes 0.56 to 0.68 ms across three runs on a shared machine, about 3% of `import nameparser`'s 18 to 24 ms in the same runs, above the 0.45 ms item 5 measured on a flat draft of 2,140 rows; the first lookup after it, 0.07 ms; compiling the whole index, 2.1 ms; uncached, the import takes about 13 to 18 ms (13 after `import nameparser` in this session, 17 to 19 in review's runs, a figure no wheel install pays). The index holds 2,157 readings over 1,798 words, 106 of them in more than one file, and 3 readings written `default = false`; it is about 95 KB of source. Recompute: `uv run python tools/vocab/build.py`, then with `PYTHONDONTWRITEBYTECODE` unset, time `import nameparser.config._index` after `import nameparser` in fresh processes (the first run writes the bytecode), and count with `from nameparser.config._index import READINGS`. - Not touched: `Lexicon` and every other pickled type gain no field, so no pickle layout moves, and 3a forces no public name; the configure-by-reading methods stay open. -- **2026-10-29 #754, sub-phase 4a — `Token.reading` IS READ AT ASSEMBLE, FROM EACH TOKEN'S FINAL ROLE AND THE TAGS THE DECIDING STAGES LEFT, ITS SUFFIX AND NAME SOURCES BY THE PREDICATES THAT NAME `Ambiguity.alternative`.** It carries out item 2 of the 2026-10-28 #754 entry above, all but `Reading.languages`, which a follow-up adds and fills from 3a's reading index (`_reading_index.readings_of`, the 2026-10-29 #753 entry above), adding the field with its values because an empty one would say that no file holds the reading. +- **2026-10-29 #754, sub-phase 4a — `Token.reading` IS READ AT ASSEMBLE, FROM EACH TOKEN'S FINAL ROLE AND THE TAGS THE DECIDING STAGES LEFT, ITS SUFFIX AND NAME SOURCES BY THE PREDICATES THAT NAME `Ambiguity.alternative`.** It carries out item 2 of the 2026-10-28 #754 entry above, all but `Reading.languages`, which a follow-up adds and fills from 3a's reading index (`_reading_index.readings_of`, the 2026-10-29 #753 entry above), adding the field with its values because an empty one would say that no file holds the reading; the follow-up landed the same day (the entry below). 1. **Where it is filled.** A word's reading is decided in two halves today: its tags say which readings of the word the vocabulary or a shape offers and how its part stands (classify's vocabulary and shape tags, the `joined` mark group and post_rules write, post_rules' unjoined marks, which make `van` in `van Gogh` a name word where `Vincent van Gogh` keeps it a particle), and the stage that gives the word its role says which one was taken. Both halves are final only at assemble, since post_rules still moves words after assign (P1's fold, P6's attachment, the patronymic rotations, `middle_as_family`'s fold), so a reading written at an earlier site would need re-writing at every later move, the stale-mark class `_types._remarked` exists to recompute. Assemble reads the suffix and name readings through `_vocab.suffix_kind` and `_vocab.name_source`, the predicates `suffix_reading` and `name_reading` use for the reading NOT taken, so the two are named by one predicate there. The bound given name's `vocabulary` and the title, connective, particle and initial readings are assemble's own tests, kept out of the shared predicates so that no alternative moves (the first draft put the bound given name in `name_source` and moved `Smith, John Abu`'s from `declared`, which review caught; `_EXPECTED_ALTERNATIVES` holds it now), the initial by classify's `initial` tag where `name_reading` names one by its shape (a one-letter `é` in `John Smith é` reads `name`, the tag never written on it). Neutral, measured 2026-10-29: the committed snapshot is byte-identical, every recorded alternative included. 2. **What each role reads as.** A title: `title` from the vocabulary where a title tag stands, by its shape where rules/H2's period-abbreviation shape claimed it (`Insp.`, `Esq. Smith`), and by position where a title phrase took a word nothing lists (`Parliament` in `Member of Parliament`); its connective `conjunction`. A suffix: `suffix_kind`'s kind and source, save a word no suffix tag marks and the vocabulary does not list, which stands there by position: a word past a second comma (`Foo` in `John Smith, Jr., Foo`), or a continuation of the entry before it (`Jones` in `John Smith PhD Jones` and in `John Smith III Jones` reads `credential`, `position`), save a credential split across a space, whose second word takes its first word's reading whole (`Ph.` `D.`); a connective reads `conjunction` from the vocabulary, as in a title or a name part, whether it joins two entries (`and` in `John Smith, PhD and MD`) or trails them with nothing to join (`John Smith, Jr. and`), as `John Smith and` does on the name side. A name-role word: `initial` by its shape where classify tagged it one, `conjunction` or `particle` from the vocabulary where it joins, and otherwise `name`, `declared` where a derived marker says so, `vocabulary` for a word the vocabulary lists as a bound given name, whether rules/P5 joined it (`Abdul` in `Abdul Aziz Tariq`) or not (`Abdul` as the middle name in `John Abdul Smith`), save one that is also a particle (`abu`), which is P5's given name only where P5 joins it, in the given part and opening its segment past the titles (`Abu Bakar Smith`, `Smith, Abu Bakar`, `Dr. Abu Bakar Smith`), and otherwise a `particle` where it joins, the chain having taken it (`John Abu Smith`, `Smith, John Abu`, and the given part of `Salim Ali abu Bakar Hassan` under FAMILY_FIRST_GIVEN_LAST, where P5 joins nothing past the family it opens with), no tag telling the two joins apart, and for a family word the segmentation split off an unspaced name by `Lexicon.surnames` (`김` in `김민준`, where a spaced `김 민준` reads `position`, the order placing it), and `position` for the rest; a particle or connective with nothing to join reads as the name word it stands as (`van` in `van Gogh`, `name`, `declared`), and so does a particle in a nickname, which has no family to join (`"Van" Smith`). 3. **Not filled**, `None`: a token `ParsedName.replace()` builds, a token `Parser.revise()` harvests, whose role the caller forced, and a token loaded from a pickle written before 3.0 (`ADDED_IN = "3.0.0"`; `tests/v2/data/parsedname_v28.pickle`, made by the released 2.8.0 wheel, loads and equals a fresh parse). `capitalized()` carries each reading through. 4. **Cost**, measured 2026-10-29 with `tools/perf/call_count.py`: the reference row moved from 305 to 306 frames on Python 3.11 and from 286 to 287 on 3.15, one frame for its suffix `III`. Every `Reading` a token can take is built once at import (280 of them, ten kinds by seven roles by four sources) and shared, where a construction per token would cost two frames a word; an untagged word, the commonest, is read in C-level tests with no frame. `_CALL_BASELINE` (307 and 288) is not moved. 5. **Left open.** A listed suffix word that the compiled sets do not split by kind reads `post-nominal`, so `Jr`, `Sr` and `Esq.` read alike: the TOML gives `jr` a `generation` reading, but `SUFFIX_WORDS` holds both kinds and classify tags them alike, so only a vocabulary lookup by kind can tell them apart: 3a's reading index, a new `Lexicon` field being set aside (the 2026-10-28 `Lexicon` surface entry makes the compiled sets internal, and `Lexicon`'s own pickle guard refuses an added field). It is the predicate's own answer, which `Ambiguity.alternative` has given since 2.7 (the snapshot's one post-nominal alternative is `Rinpoche`'s), and a kind left unset would have meant a parsed token with no reading or a kind for "not yet known", so the reading taken agrees with it until 4b, whose `generation` view needs the split, reads a listed word's kind from 3a's index; `tests/v2/pipeline/test_assemble.py::test_a_suffix_word_reads_as_its_vocabulary_kind` holds `Jr` and `Sr` as strict expected failures until then. `ReadingKind.PATRONYMIC` exists and no reading takes it until 4c, whose rules write it. This amends the 2026-10-28 #754 entry's item 9: 4b needs 3a for that kind. +- **2026-10-29 #754, 4a's follow-up — `Reading.languages` IS FILLED AT ASSEMBLE FROM 3a's READING INDEX, FOR THE READING A TOKEN TOOK AND FOR A REPORT'S ALTERNATIVE; NO PARSE MOVES.** Completes 4a (the entry above) with the field item 2 of the 2026-10-28 #754 entry names, and reverses for 3.0 the decline of decisions/A1's 2026-10-26 entry. + 1. **What it holds.** The BCP 47 tags of the shipped files holding the reading taken, on by default, each once, in tag order, read through `_reading_index.languages_of(text, categories)`, which walks the index rows without building an `IndexReading`. A reading matches a file's rows by category, not by the vocabulary's finer kind: a `title`, `particle` or `conjunction` reading its category's rows, a `name` reading the `name` and `bound_given` rows, and every suffix kind the word's suffix rows, a word having one suffix kind (the build's check 4), so `Jr`, which reads `post-nominal` until 4b, names `en`, the file of its `generation` reading. Only a `vocabulary` or `declared` reading is looked up; a shape, position, an initial or a patronymic names no file. A reading written `default = false` names none (`St` in a caller's `suffix_words`, held off by `cs.toml`), and neither does a word a caller's own `Lexicon` adds, nor the Korean surnames that divide unspaced hangul, which `config/surnames.py` keeps outside the per-word files: `김` in `김민준` reads `name`, `vocabulary`, with no files. It names the default's files whatever `Lexicon` the parse used, the semantics decisions/A1's 2026-10-26 decline weighed against it: a file holds the reading or not, and which lexicon read it is the caller's own knowledge. Under a choice of languages (3b) a "live" reading becomes one the choice makes, and `languages_of` filtering on `on_by_default` is the place 3b changes. + 2. **How a word is looked up.** By the vocabulary's own form of the word (`_normalize`); a suffix also with every period removed, as a credential is matched (`M.A` in `John Smith (M.A)`); a credential split across a space whole, its first word joined to the second (`Ph.` `D.` as `phd`), the second taking the first's reading; and a word joining listed words with punctuation by its parts, in order, a suffix's parts divided by any mark but the period first (`M.D./Ph.D.`, `MD;`), then any word's by every mark (`Lt.Gov.`), never at a combining mark, which `\W` would count (`प्रोफे.डॉ.` names `hi` and `mr` through `डॉ`). An alternative names the files for any word its report names, in order. MEASURED 2026-10-29 on `bd02d68f` plus this change over the snapshot's 6,359 distinct names (20,804 tokens): 3,935 readings are `vocabulary` or `declared`, 3,778 of them name a file, and the 157 that name none are all Korean surnames split off by the list; by the normalized form and the suffix's period-free form alone, 265 would name none, the further 108 being the words of credentials split across a space, 40 by `split:credential` (`Ph.` `D.`) and 26 by `split:member` (`Pharm D`), and 42 words joining listed words. Recompute: parse every `name` of `tools/differential/snapshot.jsonl` and count the tokens whose `reading.source` is `vocabulary` or `declared` by whether `reading.languages` is empty; for the second figure, look each up with `languages_of` by its normalized and period-free forms only, and sort the misses by the split tags on the word and the word before it. + 3. **The field.** `Reading.languages: tuple[str, ...] = ()`, `compare=False`, so two readings compare as in 2.x and every recorded alternative is untouched, and `metadata={ADDED_IN: "3.0.0"}`, `Reading` having shipped in 2.7: a `Reading` pickled by 2.7 or 2.8 loads with `()` (`tests/v2/data/parsedname_v28_alternative.pickle`, made by the released 2.8.0 wheel), and a renamed field or a shipped one missing beside it is still refused (`_REFUSED_SKEWS`). The constructor takes any iterable of str and refuses a bare str and a non-str tag. The repr names the files where there are any (`Reading('particle' as 'family', 'vocabulary', in es fr it nl pt)`). The snapshot does not record readings, so nothing re-records; it is byte-identical. + 4. **Cost**, measured 2026-10-29, Python 3.11.17. Two memos keep the lookups off the frame count: `_languages`, an `lru_cache(maxsize=4096)` keyed on the word as written and the kind, and an `lru_cache(maxsize=1024)` over `Reading` for the readings that name files, each a pure function of the shipped files, bounded because the caller's text keys them, and C-level on a hit, so a word read before costs no frame. `tools/perf/call_count.py`'s reference row moved from 311.00 on `bd02d68f` to 311.40, all of it the first reference parse of the 50 (331 frames against 311: the index import and the four first lookups); after it each parse counts 311.0 on both trees. `_CALL_BASELINE` (307) is not moved and the serial benchmark passes. In fresh processes with bytecode cached, as an installed wheel has it, the first parse to read a listed word (`Dr. Juan de la Vega III`, after a `John Smith`) takes 1.20 to 1.22 ms against 0.22 ms on `bd02d68f`, the medians of 21 processes in each of three runs: about 1.0 ms once per process, the index import 3a measured at 0.56 to 0.68 ms plus the first lookups and readings. The steady parse, over 4,000 names half of them listed-word names, takes 78.6 to 79.0 µs against 78.3 to 79.4 µs on `bd02d68f`, within the runs' noise. Recompute: time the two parses in fresh processes and a `timeit` over the 4,000 names in each tree, with `PYTHONDONTWRITEBYTECODE` unset and the index's bytecode written. + 5. **Not done here: a listed suffix word's kind.** The index lookup this change makes does not settle 4b's `Jr` and `Sr` (`tests/v2/pipeline/test_assemble.py`'s strict expected failures stand): the kind a token takes is named by `_vocab.suffix_kind`, which also names `Ambiguity.alternative`, so reading the kind from the index at assemble alone would split the two, and reading it inside `suffix_kind` moves the alternatives and puts the index on the decision path. 4b decides it. diff --git a/docs/design/vocabulary-model-rfc.md b/docs/design/vocabulary-model-rfc.md index a4bcc64ba..7df3d8e7e 100644 --- a/docs/design/vocabulary-model-rfc.md +++ b/docs/design/vocabulary-model-rfc.md @@ -219,7 +219,7 @@ The 2 broken are pairs whose romanization opens with a listed title, which reads ### Where the provenance lives at runtime -The compiled module holds sets, and a set has forgotten which file each word came from. Phase 3 compiles a second module beside it (decided 2026-10-28): a **reading index**, one row per reading (its text, its file's tag, its kind and its behavioral properties), loaded once per process on its first lookup (`_reading_index.readings_of`) (decided 2026-10-29, replacing the 2026-10-28 decision to load it with `Lexicon.default()`). Today no parse makes one, so no caller loads it. Phase 4's `Reading.languages` will be filled from it by a follow-up once 3a and 4a (#942, which adds `Token.reading` without `languages`, the follow-up adding the field with the values it fills, since an empty one would say no file holds the reading) have both merged; from then on the first parse will load it, every caller who parses will pay it once, and a caller who only imports nameparser will not. The import moves with the lookup, wherever the follow-up makes it. Filling `reading.languages` on first access to that field instead was weighed and set aside: a `Reading` holds its kind, role and source but not its word, so a lazy field would have to carry the word as a hidden key on a frozen, slotted, pickled type, to save a cost the measurement below puts at about 2% of the import. The compiler moves from tools/vocab/build.py into the package so that the build and a pack share one implementation, and a test holds the invariant that compiling the index with no choice reproduces `nameparser/config/_compiled.py` exactly. The same index is what item 13 of decisions/vocabulary-model's 2026-10-23 #752 entry says `Token.reading` needs in 3.0; 3a built it (decisions/vocabulary-model, 2026-10-29), its import measured there at 0.56 to 0.68 ms. +The compiled module holds sets, and a set has forgotten which file each word came from. Phase 3 compiles a second module beside it (decided 2026-10-28): a **reading index**, one row per reading (its text, its file's tag, its kind and its behavioral properties), loaded once per process on its first lookup (`_reading_index.readings_of`) (decided 2026-10-29, replacing the 2026-10-28 decision to load it with `Lexicon.default()`). Phase 4's `Reading.languages` is filled from it at assemble (4a's follow-up, decisions/vocabulary-model's 2026-10-29 #754 entry on it), so the first parse in a process that reads a word from the vocabulary loads it: every caller who parses such a name pays it once, about 1 ms on that parse, and a caller who only imports nameparser, or parses only words no file lists, does not. Filling `reading.languages` on first access to that field instead was weighed and set aside: a `Reading` holds its kind, role and source but not its word, so a lazy field would have to carry the word as a hidden key on a frozen, slotted, pickled type, to save a cost the measurement below puts at about 2% of the import. The compiler moves from tools/vocab/build.py into the package so that the build and a pack share one implementation, and a test holds the invariant that compiling the index with no choice reproduces `nameparser/config/_compiled.py` exactly. The same index is what item 13 of decisions/vocabulary-model's 2026-10-23 #752 entry says `Token.reading` needs in 3.0; 3a built it (decisions/vocabulary-model, 2026-10-29), its import measured there at 0.56 to 0.68 ms. Measured 2026-10-27 on `cfe8bf92`: the pool holds 2,140 readings over 1,797 words, 106 of them in more than one file. An index of their behavioral properties is about 93 KB of generated source. Imported with its bytecode cached, as an installed wheel has it, it takes about 0.45 ms (median of nine fresh processes, 2026-10-28 on `c8357e74`, Python 3.11.17), against about 20 ms for `import nameparser` cached the same way; with no cache it takes about 11 ms, the figure this section gave on 2026-10-27, measured in a shell that sets `PYTHONDONTWRITEBYTECODE`. That load is paid once per process, at the first lookup. Compiling the pool takes about 9 ms, paid once per choice and only by a caller who makes one. Reading the TOML instead costs about 39 ms. Recompute: the build's `read_dir`, `load` and `compile_pool`, timed; for the import, write the index as a module, `py_compile` it, and time its import in fresh processes beside a cached `import nameparser`. @@ -272,7 +272,7 @@ Languages with one to three candidates (`tr` `bay`, `it` `sig`, `lt` `jaun`, `ms ## Phase 4: order, shape and the output model -Status: **the output model is agreed** (2026-10-28, #754), and sub-phase 4a has landed but for `Reading.languages` (2026-10-29, decisions/vocabulary-model's 2026-10-29 #754 entry). The record is decisions/vocabulary-model's 2026-10-28 #754 entry; the output-model and work-order subsections below are the design its items describe. The first two subsections, order and shape and the rules a pack can choose, are as written before and are amended only where the design reaches them. Phase 4 is 3.0 work. The core API (`parse`, `Parser`, `Lexicon`, `Policy`, `Locale`, `ParsedName`) is frozen for 3.0 except for additions (decisions/3-0-reevaluations, 2026-10-28 #925, THE CORE API), so every change below says whether it is an addition. +Status: **the output model is agreed** (2026-10-28, #754), and sub-phase 4a has landed, `Reading.languages` with it (2026-10-29, decisions/vocabulary-model's two 2026-10-29 #754 entries). The record is decisions/vocabulary-model's 2026-10-28 #754 entry; the output-model and work-order subsections below are the design its items describe. The first two subsections, order and shape and the rules a pack can choose, are as written before and are amended only where the design reaches them. Phase 4 is 3.0 work. The core API (`parse`, `Parser`, `Lexicon`, `Policy`, `Locale`, `ParsedName`) is frozen for 3.0 except for additions (decisions/3-0-reevaluations, 2026-10-28 #925, THE CORE API), so every change below says whether it is an addition. ### Order is not shape @@ -315,7 +315,7 @@ Phase 4 also decides what a parse returns, aligned with CLDR's person-name field What decides a view is the reading each token took, so `Token.reading` comes first; `surname2` alone is read from a tag O6 writes, a rule rather than a word deciding it: -- **`Token.reading`**: a `Token` field holding the `Reading` the parse took, the same type `Ambiguity.alternative` holds the reading not taken in (decisions/A1), `None` for a token `replace()` or `revise()` builds. Its `kind` says what the word was read as, its `source` why, and a `languages` field, added to `Reading`, names the language files holding that reading, filled from phase 3's reading index (3a) and left empty for a reading no file holds (a caller's own vocabulary, a shape, position alone). The field takes `ADDED_IN` metadata, so a 2.x pickle loads with `None` (the pickle guard as widened in 2.7), and is left out of comparison (`compare=False`), so `==` between two `ParsedName`s means what it means in 2.x. `ReadingKind` gains `PATRONYMIC`, its docstring already allowing new kinds. +- **`Token.reading`**: a `Token` field holding the `Reading` the parse took, the same type `Ambiguity.alternative` holds the reading not taken in (decisions/A1), `None` for a token `replace()` or `revise()` builds. Its `kind` says what the word was read as, its `source` why, and a `languages` field, added to `Reading`, names the language files holding that reading, filled from phase 3's reading index (3a) and left empty for a reading no file holds (a caller's own vocabulary, a shape, position alone, and the Korean surname list, which stays outside the per-word files); it too takes `ADDED_IN` and is not compared, `Reading` having shipped in 2.7. The field takes `ADDED_IN` metadata, so a 2.x pickle loads with `None` (the pickle guard as widened in 2.7), and is left out of comparison (`compare=False`), so `==` between two `ParsedName`s means what it means in 2.x. `ReadingKind` gains `PATRONYMIC`, its docstring already allowing new kinds. - **Generation and credentials** (#326): `generation`, the suffix tokens whose reading is a generation or an agnomen (`Jr.`, `III`, `Filho`), and `credentials`, the rest of the suffix (`PhD`, `Esq.`, the honorifics after a name), so the two partition `suffix` and `suffix` is unchanged. CLDR has only these two buckets; the post-nominals that are neither (`Esq.`, the CJK and Hebrew honorifics, `Rinpoche`) fall in `credentials`, CLDR's catch-all, and a caller who wants them apart reads `Token.reading.kind`, which says `post-nominal`. Measured 2026-10-28 on `c8357e74`, over the committed corpus's 6,533 distinct entries (`snapshot.corpus_entries()`) each parsed under its declared order: of the 1,054 with a suffix, 33 hold a generation (or agnomen) and a listed credential together, 77 a generation beside any other suffix word, and 64 only listed post-nominals of other kinds. Of the 615 CLDR renderings with a generation or credentials field, 75 parse with exactly CLDR's suffix words, and in all 75 a split by the vocabulary's reading kind gives CLDR's `generation` and `credentials` both; the other 540 differ in which words the suffix holds, a parse question the split cannot fix. Recompute: classify each suffix word (a token tagged `joined` merged into the one before it) by the kinds of its suffix readings in nameparser/vocab/, matched lowercased with its periods and spaces removed, skipping `default = false` readings, a generation or agnomen reading winning over a credential and a credential over the other kinds, an unlisted roman numeral `i` to `x` a generation and any other unlisted word unlisted; over the corpus entries above, and over `cldr_fields.jsonl` for CLDR. - **A patronymic** (decision 8): `patronymic`, the tokens whose reading is `PATRONYMIC`, in whichever role the parse put them, with `family` and `middle` unchanged. What fills it is a word or a rule that says patronymic, never a guess from position: the vocabulary's `patronymic` particles (`bin`, `binti`, `ibn`, `bint`, Hebrew `bar` and `bat` and the native-script forms) with the name after them, in the default, so `Mahathir bin Mohamad` reads family `bin Mohamad` as today and patronymic `bin Mohamad` beside it; the East Slavic and Turkic rules where RU or TR_AZ turns them on, so `Ivan Ivanovich Ivanov` under RU keeps middle `Ivanovich` and adds patronymic `Ivanovich`; and an Icelandic ending rule an `is` pack would turn on, the default unable to tell `Karlsdóttir` from the Swedish surname `Johansson`. The name `patronymic` is kept for a matronymic too, which is written the same way and cannot be told from one. **`family` is not emptied** by recognizing a patronymic, revising what this section said before: the people filling in the fielded corpora put the patronymic in the family box (below), as CLDR's samples put an Icelandic patronymic in `surname` or `surname-core` (CLDR's `is` file's 3 Icelandic samples, in all 9 of their renderings; its one foreign sample puts a `-dóttir` word in `surname2`) and a Russian one in `given2` (its 2 `ru` samples, in all 6 renderings), our `middle`, while Wikidata records an Icelandic name as a patronymic with no family name; a `family` that keeps it and a `patronymic` that names it satisfy both. A caller writing "Mr. {family}" reads `patronymic` to see that the family is one. - **A second surname**: `surname2`, the family's second name unit where a rule has read the family as two (rules/O6, which a caller or an `es` pack turns on), and empty otherwise, `family` keeping both units as O6 makes it today. O6 already knows which unit it took, so it marks where the second one starts with a new stable tag; a family comma alone marks nothing, since `Lloyd Webber, Andrew` is not two surnames, and a hyphenated family is one token and stays whole. Whether the first unit gets a view of its own, and its name (`surname` sits close to the `surnames` that exists), is a value question below. `surnames` keeps its meaning, `middle` and `family` joined, the reading of a double surname the default splits. @@ -385,7 +385,7 @@ Each sub-phase is one PR, measured on its own base as phase 2's and phase 3's ar | Sub-phase | What | Needs | |---|---|---| -| 4a | `Token.reading` and `ReadingKind.PATRONYMIC`, kind and source read at assemble from each token's final role and the tags the deciding stages left; `Reading.languages` once 3a's index exists. Landed but for `languages` (decisions/vocabulary-model's 2026-10-29 #754 entry) | — (neutral); 3a for `languages` | +| 4a | `Token.reading` and `ReadingKind.PATRONYMIC`, kind and source read at assemble from each token's final role and the tags the deciding stages left; `Reading.languages` once 3a's index exists. Landed (decisions/vocabulary-model's two 2026-10-29 #754 entries) | — (neutral); 3a for `languages` | | 4b | `generation` and `credentials` (#326); a listed suffix word's kind read from 3a's index, which 4a could not split (`Jr` and `Esq.` both read `post-nominal` there) | 4a (neutral); 3a for the kind | | 4c | `patronymic`: the vocabulary's patronymic particles, and the RU and TR_AZ rules writing the reading | 4a (neutral) | | 4d | `surname2`: O6's stable tag | — (neutral) | @@ -400,7 +400,7 @@ Each sub-phase is one PR, measured on its own base as phase 2's and phase 3's ar A shape question, decided 2026-10-28: -- **Whether `Lexicon` keeps a public field for each compiled set**, or answers word, kind and property queries from phase 3's reading index and keeps its sets internal. `dataclasses.fields(Lexicon)` gives 22 public fields on 2026-10-28, 11 of them added since 2.0.0's 11 (`surnames`, `honorific_tails`, `conjunctions_ambiguous`, `after_given_titles`, `suffix_words_ambiguous`, `agnomens`, `given_name_abbreviations`, `titles_ambiguous`, `title_modifiers`, `title_styles`, `title_phrases`), each a new parse behavior's set, and a rule keyed on a title kind or on one language's reading of a kind (phase 3's design, "a property a language's own reading should take only under its pack") would add more. Keeping the fields: each is frozen API, shown in the bounded repr (`Lexicon(default + titles: +1)`), and pickled, where `Lexicon`'s own `__setstate__`, unlike the `ADDED_IN` guard the other types share, refuses any pickle whose fields differ, so every new field breaks every older `Lexicon` pickle, and with it every pickled `Parser` and locale pack, until that guard learns the marker. Turning them internal: a break that has to land in 3.0, where a query added beside the fields is an addition that can come later; a caller who configures vocabulary by the field today (`Lexicon.add(titles=...)`, `dataclasses.replace`) needs another way in, for words that carry no kind and no language; and phase 3's reading index, which a `Lexicon` answering queries would read, is loaded on its first lookup (decided lazy on 2026-10-29 rather than loaded with `Lexicon.default()`; about 0.6 ms with its bytecode cached, by phase 3's timing above). No parse makes one today; once a follow-up to 4a fills `Reading.languages`, every parse will, so answering queries from it will add no load beyond that first lookup. The parser need not pay either way: a frozenset test per word costs no frame and an internal cache keeps that path, while a query method called per word would add one per token, the cost the frame budget counts as scaling with the input. **Derek decided on 2026-10-28 that a caller configures vocabulary by reading and the compiled sets become internal** (decisions/vocabulary-model's 2026-10-28 #754 entry on the `Lexicon` surface). A caller states what a word may be in the language files' terms, a `given` reading on `guru`, no credential reading on `dc`, and `Lexicon` derives the sets and gates the build derives, so making `Guru` a possible given name needs no knowledge of rules/H9's sets, where today it is `remove(titles={'guru'}).add(titles_ambiguous={'guru'})`, and `add(given_name_titles=...)` or `add(suffix_acronyms_ambiguous=...)`, the plausible wrong guesses, do something else. A reading is also what a caller can know about a word and what a pack contribution states. Removing a reading from a caller's own lexicon may settle a clash; decision 15 binds the default and the packs, not a caller's choice about their own data. It breaks the frozen `Lexicon` (decisions/3-0-reevaluations); the method names and whether a caller's own language file can join the pool later (phase 3 defers it) are open, and the 2.x warning the freeze asks for was waived on 2026-10-29, no 2.x feature release being left to carry it (decisions/3-0-reevaluations, THE LEXICON WARNING). +- **Whether `Lexicon` keeps a public field for each compiled set**, or answers word, kind and property queries from phase 3's reading index and keeps its sets internal. `dataclasses.fields(Lexicon)` gives 22 public fields on 2026-10-28, 11 of them added since 2.0.0's 11 (`surnames`, `honorific_tails`, `conjunctions_ambiguous`, `after_given_titles`, `suffix_words_ambiguous`, `agnomens`, `given_name_abbreviations`, `titles_ambiguous`, `title_modifiers`, `title_styles`, `title_phrases`), each a new parse behavior's set, and a rule keyed on a title kind or on one language's reading of a kind (phase 3's design, "a property a language's own reading should take only under its pack") would add more. Keeping the fields: each is frozen API, shown in the bounded repr (`Lexicon(default + titles: +1)`), and pickled, where `Lexicon`'s own `__setstate__`, unlike the `ADDED_IN` guard the other types share, refuses any pickle whose fields differ, so every new field breaks every older `Lexicon` pickle, and with it every pickled `Parser` and locale pack, until that guard learns the marker. Turning them internal: a break that has to land in 3.0, where a query added beside the fields is an addition that can come later; a caller who configures vocabulary by the field today (`Lexicon.add(titles=...)`, `dataclasses.replace`) needs another way in, for words that carry no kind and no language; and phase 3's reading index, which a `Lexicon` answering queries would read, is loaded on its first lookup (decided lazy on 2026-10-29 rather than loaded with `Lexicon.default()`; about 0.6 ms with its bytecode cached, by phase 3's timing above). Since 4a's follow-up filled `Reading.languages` (2026-10-29), the first parse that reads a listed word makes one, so answering queries from it adds no load beyond that first lookup. The parser need not pay either way: a frozenset test per word costs no frame and an internal cache keeps that path, while a query method called per word would add one per token, the cost the frame budget counts as scaling with the input. **Derek decided on 2026-10-28 that a caller configures vocabulary by reading and the compiled sets become internal** (decisions/vocabulary-model's 2026-10-28 #754 entry on the `Lexicon` surface). A caller states what a word may be in the language files' terms, a `given` reading on `guru`, no credential reading on `dc`, and `Lexicon` derives the sets and gates the build derives, so making `Guru` a possible given name needs no knowledge of rules/H9's sets, where today it is `remove(titles={'guru'}).add(titles_ambiguous={'guru'})`, and `add(given_name_titles=...)` or `add(suffix_acronyms_ambiguous=...)`, the plausible wrong guesses, do something else. A reading is also what a caller can know about a word and what a pack contribution states. Removing a reading from a caller's own lexicon may settle a clash; decision 15 binds the default and the packs, not a caller's choice about their own data. It breaks the frozen `Lexicon` (decisions/3-0-reevaluations); the method names and whether a caller's own language file can join the pool later (phase 3 defers it) are open, and the 2.x warning the freeze asks for was waived on 2026-10-29, no 2.x feature release being left to carry it (decisions/3-0-reevaluations, THE LEXICON WARNING). Value questions inside the shapes above: diff --git a/docs/release_log.rst b/docs/release_log.rst index fe0d6dc5e..0d569f323 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -7,6 +7,7 @@ Release Log **Additions** - **Add Token.reading, the reading the parse took for each word.** ``parse("John Smith PhD Jones").tokens[-1].reading`` is ``Reading('credential' as 'suffix', 'position')``, the reading 2.7's ``Ambiguity.alternative`` gave only for the other side of a fork, and ``ReadingKind`` gains ``patronymic``. The field is not compared, and a ``ParsedName`` pickled by 2.x loads with no readings. :issue:`754` + - **Add Reading.languages, the language files that list a word the way it was read.** ``parse("Dr. Juan de la Vega III").tokens[2].reading.languages`` is ``('es', 'fr', 'it', 'nl', 'pt')`` for the particle ``de``, and ``Ambiguity.alternative`` names its files too; a reading by shape or position names none. It is not compared, and a ``Reading`` pickled by 2.7 or 2.8 loads with none. The first parse in a process that meets a listed word loads the reading index, about 1 ms once. :issue:`754`, :issue:`753` **Breaking Changes** diff --git a/docs/usage.rst b/docs/usage.rst index 9d55f884b..eea49e461 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1124,7 +1124,8 @@ The reading it did not take Since 2.7 a report of a fork also says what the other reading was. ``Ambiguity.alternative`` is a :class:`~nameparser.Reading` with three -parts: +parts, and since 3.0 a fourth, its ``languages`` (see +:ref:`reading-taken`): - **kind** — what the word would have been read as: a ``name``, an ``initial``, a ``credential``, a ``generation``, a ``particle``, a @@ -1194,6 +1195,26 @@ otherwise have been the family name: >>> jones_run.ambiguities[0].alternative Reading('name' as 'family', 'position') +A reading from the vocabulary also names the language files that list +the word that way, as BCP 47 tags in ``languages``. A reading by shape +or position names none, and neither does a word from your own +``Lexicon`` that no shipped file lists: + +.. doctest:: + + >>> for tok in parse("Dr. Juan de la Vega III").tokens: + ... print(f"{tok.text}: {tok.reading.languages}") + Dr.: ('mul',) + Juan: () + de: ('es', 'fr', 'it', 'nl', 'pt') + la: ('es', 'fr', 'it') + Vega: () + III: ('mul',) + +``mul`` is the file of words shared across languages on purpose. The +first parse in a process that meets a listed word loads the index of +those files, about a millisecond once. + A token the parse did not read has no reading: one that :meth:`~nameparser.ParsedName.replace` or :meth:`~nameparser.Parser.revise` builds, and one loaded from a pickle diff --git a/nameparser/_pipeline/AGENTS.md b/nameparser/_pipeline/AGENTS.md index 3297a703f..0d4bec9d5 100644 --- a/nameparser/_pipeline/AGENTS.md +++ b/nameparser/_pipeline/AGENTS.md @@ -44,7 +44,7 @@ And work a report does must not walk the name: the given-name title test, asked ### The reading taken is read at assemble -`Token.reading` (3.0, #754's 4a) is not written by the deciding stages: assemble reads it from each token's final role and the tags the deciding stages left (classify's, the `joined` mark group and post_rules write, post_rules' unjoined marks), since a reading written where a word first takes its role would go stale at every later move (P1's fold, P6's attachment, the policy folds). The suffix and name halves ask `_vocab.suffix_kind` and `_vocab.name_source`, the predicates `suffix_reading` and `name_reading` use for the reading not taken, so a change to which reading a TAG offers changes both through one function there; the bound given name's source and the title, connective, particle and initial readings are assemble's own tests, kept out of the shared predicates because an arm added there moves every alternative they name (4a's first draft moved `Smith, John Abu`'s), the initial read off the `initial` tag where `name_reading` tests the shape. A new tag that says what a word was read as, or a decision whose outcome no tag and no role records, needs assemble taught too, with a row in `tests/v2/pipeline/test_assemble.py::_EXPECTED_READINGS`, which holds one row per branch assemble names a reading by; `test_every_parsed_token_has_a_reading_of_its_own_field` checks every case row has one, in its own field, and keeps it through `capitalized()`. decisions/vocabulary-model's 2026-10-29 #754 entry has which role reads as which reading. +`Token.reading` (3.0, #754's 4a) is not written by the deciding stages: assemble reads it from each token's final role and the tags the deciding stages left (classify's, the `joined` mark group and post_rules write, post_rules' unjoined marks), since a reading written where a word first takes its role would go stale at every later move (P1's fold, P6's attachment, the policy folds). The suffix and name halves ask `_vocab.suffix_kind` and `_vocab.name_source`, the predicates `suffix_reading` and `name_reading` use for the reading not taken, so a change to which reading a TAG offers changes both through one function there; the bound given name's source and the title, connective, particle and initial readings are assemble's own tests, kept out of the shared predicates because an arm added there moves every alternative they name (4a's first draft moved `Smith, John Abu`'s), the initial read off the `initial` tag where `name_reading` tests the shape. A new tag that says what a word was read as, or a decision whose outcome no tag and no role records, needs assemble taught too, with a row in `tests/v2/pipeline/test_assemble.py::_EXPECTED_READINGS`, which holds one row per branch assemble names a reading by; `test_every_parsed_token_has_a_reading_of_its_own_field` checks every case row has one, in its own field, and keeps it through `capitalized()`. decisions/vocabulary-model's 2026-10-29 #754 entry has which role reads as which reading. A reading's `languages` is assemble's too, looked up in the reading index by the word and the kind taken, never by a deciding stage: a new kind needs a row in `_assemble._CATEGORIES` naming the vocabulary categories that hold it, or it names no file, and a new way of writing a listed word (a split, a join) needs a row in `_EXPECTED_LANGUAGES`; the entry after that one has how a word is looked up. ### Testing an emitter diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py index ba1fa4727..375276a1c 100644 --- a/nameparser/_pipeline/_assemble.py +++ b/nameparser/_pipeline/_assemble.py @@ -15,7 +15,9 @@ policy folds a middle; Policy.name_order, for ParsedName.order where no positional read happened; and Lexicon.surnames, suffix_acronyms and suffix_acronyms_ambiguous, for the source of a token's reading where -no tag gives it. +no tag gives it. Not a Lexicon: the shipped reading index +(_reading_index.languages_of), for Reading.languages, which names the +files and so is the same under every Lexicon. Structural tokens (dropped maiden markers) are omitted, like delimiter characters. A main-stream token that somehow reaches here with no role @@ -29,6 +31,8 @@ """ from __future__ import annotations +import functools +import unicodedata from types import MappingProxyType from nameparser._lexicon import Lexicon, _normalize @@ -41,6 +45,7 @@ from nameparser._pipeline._vocab import ( _PERIOD_ABBREV, name_source, suffix_kind, ) +from nameparser._reading_index import languages_of from nameparser._types import ( FOLDED_TAG, GIVEN_FIRST, UNJOINED_CONJUNCTION_TAG, UNJOINED_TAG, Ambiguity, AmbiguityKind, ParsedName, Reading, ReadingKind, @@ -55,6 +60,72 @@ (kind, role, source): Reading(kind, role, source) for kind in ReadingKind for role in Role for source in ReadingSource}) +#: The vocabulary categories (_reading_index.CATEGORY's values) whose +#: rows hold a reading of each kind. A suffix reading takes the word's +#: suffix rows whatever their finer kind, since a word has one suffix +#: kind (the build's check 4) and the parse does not yet read it (4b); +#: a name, its name and bound given name rows. An initial and a +#: patronymic are read by shape or rule, never from a file. +_CATEGORIES: MappingProxyType[ReadingKind, frozenset[str]] = MappingProxyType({ + ReadingKind.TITLE: frozenset({"title"}), + ReadingKind.PARTICLE: frozenset({"particle"}), + ReadingKind.CONJUNCTION: frozenset({"conjunction"}), + ReadingKind.CREDENTIAL: frozenset({"suffix"}), + ReadingKind.GENERATION: frozenset({"suffix"}), + ReadingKind.AGNOMEN: frozenset({"suffix"}), + ReadingKind.POST_NOMINAL: frozenset({"suffix"}), + ReadingKind.NAME: frozenset({"name", "bound_given"}), +}) + +#: The sources a file can hold: a listed word, and a name the +#: vocabulary declares beside another reading. +_LISTED = frozenset({ReadingSource.VOCABULARY, ReadingSource.DECLARED}) + + +# Both memos are C-level on a hit, so a word read before costs no +# frame (decisions/parse-cost), and both are pure functions of the +# shipped files, bounded since what keys them comes from the caller's +# text: `_languages` of a word as written, `_reading` of the readings +# that name files. +@functools.lru_cache(maxsize=4096) +def _languages(text: str, kind: ReadingKind) -> tuple[str, ...]: + """The files holding the `kind` reading of `text` as written, + Reading.languages: by the vocabulary's own form of the word; a + suffix also with every period removed, the form a credential is + matched by ('M.A' in 'John Smith (M.A)'); and a word joining + listed words with punctuation ('Lt.Gov.', 'MD/PhD', 'MD;') by + the files of its parts, in order.""" + cats = _CATEGORIES.get(kind) + if cats is None: + return () + n = _normalize(text) + acronym = "suffix" in cats + found = languages_of(n, cats) + if not found and acronym and "." in n: + found = languages_of(n.replace(".", ""), cats) + if not found and acronym: + # its parts by any other mark, each an acronym ('M.D./Ph.D.') + found = _by_parts(n, cats, ".") + return found or _by_parts(n, cats, "") + + +def _by_parts(n: str, cats: frozenset[str], keep: str) -> tuple[str, ...]: + """The files of the parts of `n`, in order: divided by punctuation, + symbols and spaces, but for the marks in `keep`, which are removed + from each part instead; never by a combining mark, which a `[^\\w]` + would count ('डॉ' in 'प्रोफे.डॉ.').""" + parts = "".join(" " if c not in keep + and unicodedata.category(c)[0] in "PSZ" else c + for c in n).split() + return tuple(dict.fromkeys( + tag for part in parts + for tag in languages_of(part.replace(keep, "") if keep else part, + cats))) + + +_reading = functools.lru_cache(maxsize=1024)(Reading) + + #: The tags that say a list or a shape made a word a suffix: without #: one, a suffix word stands there by position, unless the vocabulary #: lists it (a bracketed member, which classify leaves untagged). @@ -76,7 +147,8 @@ def _alternative(pending: PendingAmbiguity, materialized: tuple[Token, ...], """The public Reading of a report's PendingReading, its field named from the final roles where the emitter deferred it (O5's other primary field) and folded as `middle_as_family` folds every middle - (rules/O6's neighbour, the policy's own fold).""" + (rules/O6's neighbour, the policy's own fold), and its languages + the files holding it for any word the report names.""" alt = pending.alternative if alt is None or not materialized: return None @@ -94,7 +166,12 @@ def _alternative(pending: PendingAmbiguity, materialized: tuple[Token, ...], return None if middle_as_family and role is Role.MIDDLE: role = Role.FAMILY - return Reading(alt.kind, role, alt.source) + # the files holding the reading for any of its words, as a token's + # own reading names them + langs = (tuple(dict.fromkeys( + tag for t in materialized for tag in _languages(t.text, alt.kind))) + if alt.source in _LISTED else ()) + return Reading(alt.kind, role, alt.source, langs) def _opens_its_segment(state: ParseState, i: int) -> bool: @@ -155,9 +232,11 @@ def assemble(state: ParseState) -> ParsedName: # `_listed_suffix` and its `_normalize`). `_vocab`'s two predicates # also name the sources of the reading NOT taken # (Ambiguity.alternative). - # Reading.languages joins here, in 4a's follow-up: phase 3's - # reading index (`_reading_index.readings_of`, 3a) answers - # which files hold a reading. + # Reading.languages, the files holding the reading taken, is + # read below from phase 3's reading index for a listed or + # declared reading, through `_languages`, whose hit costs no + # frame (decisions/vocabulary-model, 2026-10-29). + reading: Reading | None = None if role is Role.SUFFIX: kind, source = suffix_kind(t.text, tags) if (source is ReadingSource.VOCABULARY @@ -175,12 +254,22 @@ def assemble(state: ParseState) -> ParsedName: # the second word of a credential split across # a space, which is one credential ('Ph.' 'D.'); # any other word a run took stays a credential - # by position ('Jones' in 'John Smith III Jones') - kind, source = before.kind, before.source - reading = _READINGS[kind, role, source] - before = reading + # by position ('Jones' in 'John Smith III Jones'), + # files and all + reading = before prev_split = (SPLIT_CREDENTIAL_TAG in tags or SPLIT_MEMBER_TAG in tags) + if reading is None: + # a credential split across a space is listed whole + # ('Ph.' 'D.' as 'phd'): its first word reads the files + # of the two words, which the second then takes + written = (t.text + tokens[i + 1].text + if prev_split and i + 1 < last else t.text) + reading = (_reading(kind, role, source, langs) + if source in _LISTED + and (langs := _languages(written, kind)) + else _READINGS[kind, role, source]) + before = reading else: before = None if role is Role.TITLE: @@ -238,7 +327,10 @@ def assemble(state: ParseState) -> ParsedName: # a particle or connective with nothing to join reads # as the name word it stands as ('van' in 'van Gogh') kind, source = ReadingKind.NAME, name_source(tags) - reading = _READINGS[kind, role, source] + reading = (_reading(kind, role, source, langs) + if source in _LISTED + and (langs := _languages(t.text, kind)) + else _READINGS[kind, role, source]) final[i] = Token(t.text, t.span, role, tags, reading) # No alphanumeric character among the SURVIVING tokens means no # name: a bare '.' or '- -' is not a person. v1 kept such input diff --git a/nameparser/_reading_index.py b/nameparser/_reading_index.py index a37114ab8..0f70b537e 100644 --- a/nameparser/_reading_index.py +++ b/nameparser/_reading_index.py @@ -17,7 +17,9 @@ of languages run, so there is one implementation of what a reading compiles to; tests/v2/test_reading_index.py holds that compiling the index reproduces _compiled.py exactly. It is also the one way to read -the index at run time (:func:`readings_of`, :func:`index`). +the index at run time (:func:`readings_of`, :func:`index`), and the +reader every parse calls for ``Reading.languages`` +(:func:`languages_of`), so the first parse in a process loads it. It imports nothing internal at module level, and the index only inside :func:`index`: the build loads this file by path, as it loads @@ -165,8 +167,8 @@ class IndexReading(ReadingProperties): A lookup result, not a value to keep: its ``props`` is a read-only view, so an IndexReading neither hashes nor pickles. What a parse - would keep of it (``Reading.languages``, filled by a follow-up to - phase 4a) is its tag.""" + keeps of it is its tag, in ``Reading.languages`` + (:func:`languages_of`).""" text: str tag: str @@ -231,11 +233,27 @@ def readings_of(text: str) -> tuple[IndexReading, ...]: edge periods removed; _lexicon._normalize), and a phrase whole (``"z domu"``). A reading the parse took from a caller's own vocabulary, from a shape or from position has no row here, which is - what ``Reading.languages`` is to read as empty once a follow-up to - phase 4a fills it from here; no parse calls this yet.""" + what an empty ``Reading.languages`` says; the parse reads the tags + through :func:`languages_of`, which builds no IndexReading.""" return tuple(from_row(text, row) for row in _by_text().get(text, ())) +def languages_of(text: str, categories: frozenset[str]) -> tuple[str, ...]: + """The tags of the files holding a reading of `text`, on by + default, whose kind falls in one of `categories` (values of + :data:`CATEGORY`), each once, in index order; empty for a word no + file holds in them. `text` is normalized as :func:`readings_of` + takes it. What assemble fills ``Reading.languages`` with + (decisions/vocabulary-model, 2026-10-29); its first call in a + process loads the index.""" + out: list[str] = [] + for tag, kind, pairs in _by_text().get(text, ()): + if (CATEGORY[kind] in categories and tag not in out + and ("default", False) not in pairs): + out.append(tag) + return tuple(out) + + class CompileError(ValueError): """The readings cannot compile: a fact about the data, which the build reports as a vocabulary error (exit 2) where any other diff --git a/nameparser/_types.py b/nameparser/_types.py index d298af538..dff34aecd 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -949,6 +949,18 @@ class Reading: #: Where the reading comes from: a listed word, a declared name, a #: shape, or position alone. source: ReadingSource + #: The BCP 47 tags of the shipped vocabulary files that hold this + #: reading of the word, on by default (``("es", "fr", "it", "pt")`` + #: for the particle ``de``, ``("mul",)`` for the title ``Dr.``), in + #: tag order. Empty for a reading no file holds: a shape, position + #: alone, a caller's own vocabulary, or a shipped list kept outside + #: the per-word files (the Korean surnames that divide ``김민준``). + #: A suffix reading is matched by the word's suffix entry whatever + #: its finer kind, a word having one. An ``Ambiguity.alternative`` + #: names the files holding it for any word the report names. Not + #: compared: two readings are equal without it, as in 2.x. + languages: tuple[str, ...] = field( + default=(), compare=False, metadata={ADDED_IN: "3.0.0"}) # in the class body so @dataclass(slots=True) keeps them __getstate__ = _guarded_getstate @@ -957,6 +969,18 @@ class Reading: def __post_init__(self) -> None: # the members the parser passes need no coercion, and a report # pays no frame for it (decisions/parse-cost) + if type(self.languages) is not tuple: + if isinstance(self.languages, str): + raise TypeError( + f"Reading.languages must be an iterable of BCP 47 " + f"tags, got the str {self.languages!r}; write " + f"({self.languages!r},)") + object.__setattr__(self, "languages", tuple(self.languages)) + for tag in self.languages: + if not isinstance(tag, str): + raise TypeError( + f"Reading.languages must hold BCP 47 tags as str, got " + f"{tag!r} in {self.languages!r}") if not isinstance(self.kind, ReadingKind): object.__setattr__( self, "kind", @@ -971,8 +995,10 @@ def __post_init__(self) -> None: "sources")) def __repr__(self) -> str: + langs = (f", in {' '.join(self.languages)}" if self.languages + else "") return (f"Reading({self.kind.value!r} as {self.role.value!r}, " - f"{self.source.value!r})") + f"{self.source.value!r}{langs})") @dataclass(frozen=True, slots=True) diff --git a/tests/v2/data/parsedname_v28_alternative.pickle b/tests/v2/data/parsedname_v28_alternative.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b7d15c54bb2d60f399674f48623c8a1f10b51dce GIT binary patch literal 685 zcmZ8fO-~y!5RK3#e1xD@h^oC=RVCmMZoRap+<;YRm2j`TS!Zpru~+tNAO|Ek1TMay zO#eWCX~*6Ws;;!!@x1qD=gs&b`Tb|B6(9dz^O>l)b;6$gf<+}9`n{jwoL>=wD?EL~ zdpz+w#;QVTUZU?^{y4v4B>h4wIKhDjBXWwEAeV%1j%oPV^H2Mz>)Cl@~ zmZ_pPwZqWdGx~0mQJ`p&5xT3+^ z1Y=$)cww^;FNYm zZvMzD2PrHANn9RDjIITs+?HRK!B+{wY8t{k?rD2O1(Yr}uH;uT)Lmy^?Iar$w3*Mo g<4kQQQQuoL$4!Q;@8n`qax3y^y3XosNh3Y`3&l?b1ONa4 literal 0 HcmV?d00001 diff --git a/tests/v2/pipeline/test_assemble.py b/tests/v2/pipeline/test_assemble.py index 2825c038c..15f6e86fd 100644 --- a/tests/v2/pipeline/test_assemble.py +++ b/tests/v2/pipeline/test_assemble.py @@ -1,6 +1,8 @@ import pytest -from nameparser import Parser, Reading, ReadingKind, locales, parse, parser_for +from nameparser import ( + Parser, Reading, ReadingKind, ReadingSource, locales, parse, parser_for, +) from nameparser._lexicon import Lexicon from nameparser._pipeline import run from nameparser._pipeline._assemble import assemble @@ -351,6 +353,9 @@ def test_every_parsed_token_has_a_reading_of_its_own_field(case: Case) -> None: assert t.reading.role is t.role, t # 4c's: no rule of the parse names a patronymic yet assert t.reading.kind is not ReadingKind.PATRONYMIC, t + # no file holds a reading by shape or by position + if t.reading.source in (ReadingSource.SHAPE, ReadingSource.POSITION): + assert t.reading.languages == (), t # case repair changes case and nothing else, the readings included repaired = parser.capitalized(pn, force=True) assert ([t.reading for t in repaired.tokens] @@ -411,3 +416,78 @@ def test_a_suffix_word_reads_as_its_vocabulary_kind( (tok,) = [t for t in parse(text).tokens if t.text == word] assert tok.reading is not None assert tok.reading.kind.value == kind + + +# -- Reading.languages (3.0, 4a's follow-up): the files holding it ------ + +#: (name, word, languages): the files the reading index holds the +#: reading the parse took in, one row per way assemble looks a word up. +_EXPECTED_LANGUAGES = [ + # a particle, by its particle rows only ('la' is a given name in es + # too, a reading the parse did not take) + ("Dr. Juan de la Vega III", "de", ("es", "fr", "it", "nl", "pt")), + ("Dr. Juan de la Vega III", "la", ("es", "fr", "it")), + ("Dr. Juan de la Vega III", "Dr.", ("mul",)), + ("Dr. Juan de la Vega III", "III", ("mul",)), + ("John Smith PhD", "PhD", ("en",)), + ("王小明先生", "先生", ("ja", "zh")), + ("Juan e Maria Smith", "e", ("it", "pt")), + ("Pedro Alves Filho", "Filho", ("pt",)), + # a suffix matched by its suffix entry, whatever its finer kind (4b) + ("John Smith Jr.", "Jr.", ("en",)), + # a name the vocabulary declares, and a bound given name + ("van Gogh", "van", ("nl",)), + ("Chas. Smith", "Chas.", ("en",)), + ("Abu Bakar Smith", "Abu", ("ar",)), + # an acronym by its periods removed, and a credential split across + # a space, listed whole, each of its words naming the files + ("John Smith (M.A)", "M.A", ("en",)), + ("John Smith Ph. D.", "Ph.", ("en",)), + ("John Smith Ph. D.", "D.", ("en",)), + ("Smith, Ph. D. III", "D.", ("en",)), + ("John Smith Pharm D", "Pharm", ("en",)), + # a word joining listed words, by its parts; never divided at a + # combining mark + ("John Smith, M.D./Ph.D.", "M.D./Ph.D.", ("en",)), + ("Lt.Gov. John Smith", "Lt.Gov.", ("en",)), + ("प्रोफे.डॉ. अदा शर्मा", "प्रोफे.डॉ.", ("hi", "mr")), + # no file: position, shape, and a shipped list outside the per-word + # files (the Korean surnames) + ("Dr. Juan de la Vega III", "Juan", ()), + ("John Smith X.Y.Z.", "X.Y.Z.", ()), + ("Insp. Jane Morse", "Insp.", ()), + ("John Q. Smith", "Q.", ()), + ("김민준", "김", ()), + ("John Smith Ph. D. Jones", "Jones", ()), +] + + +@pytest.mark.parametrize("text,word,languages", _EXPECTED_LANGUAGES, + ids=lambda v: v if isinstance(v, str) else None) +def test_a_reading_names_the_files_holding_it( + text: str, word: str, languages: tuple[str, ...]) -> None: + (tok,) = [t for t in parse(text).tokens if t.text == word][:1] + assert tok.reading is not None + assert tok.reading.languages == languages + + +def test_a_reading_from_a_callers_own_vocabulary_names_no_file() -> None: + # a word no file holds, and one a file holds only off by default + # (`st`, a Czech generation written `default = false`) + lex = Lexicon.default().add(titles={"zzq"}, suffix_words={"st"}) + pn = Parser(lexicon=lex).parse("Zzq John Smith St") + readings = {t.text: t.reading for t in pn.tokens} + assert readings["Zzq"] == Reading("title", "title", "vocabulary") # type: ignore[arg-type] + assert readings["St"] == Reading("post-nominal", "suffix", "vocabulary") # type: ignore[arg-type] + assert readings["Zzq"] is not None and readings["Zzq"].languages == () + assert readings["St"] is not None and readings["St"].languages == () + + +def test_an_alternative_names_the_files_holding_it() -> None: + (amb,) = parse("JOHN SMITH MA").ambiguities + assert amb.alternative is not None + # `ma` as a name: given and family in en, given in fil + assert amb.alternative.languages == ("en", "fil") + (amb,) = parse("van Gogh").ambiguities + assert amb.alternative is not None + assert amb.alternative.languages == ("nl",) diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 14b7657f3..787853291 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -112,7 +112,11 @@ "_pipeline/_comma.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_assign.py": _PIPELINE_STAGE_ALLOWED, "_pipeline/_post_rules.py": _PIPELINE_STAGE_ALLOWED, - "_pipeline/_assemble.py": _PIPELINE_STAGE_ALLOWED, + # not a stage: it alone reads the reading index, for + # Reading.languages (4a's follow-up); no stage that decides a + # role may, since the index is the files' record, not the lexicon + "_pipeline/_assemble.py": _PIPELINE_STAGE_ALLOWED + ( + "nameparser._reading_index",), # Parser sits on everything except _render "_parser.py": ("nameparser._types", "nameparser._lexicon", "nameparser._policy", "nameparser._locale", diff --git a/tests/v2/test_reprs.py b/tests/v2/test_reprs.py index 3675134e8..c705762ea 100644 --- a/tests/v2/test_reprs.py +++ b/tests/v2/test_reprs.py @@ -9,7 +9,8 @@ from nameparser._parser import Parser, parse from nameparser._policy import FAMILY_FIRST, PatronymicRule, Policy, PolicyPatch from nameparser._types import ( - Ambiguity, AmbiguityKind, ParsedName, Role, Segmentation, Span, Token, + Ambiguity, AmbiguityKind, ParsedName, Reading, Role, Segmentation, Span, + Token, ) @@ -31,7 +32,10 @@ def test_ambiguity_repr_shows_kind_and_token_texts() -> None: def test_reading_repr_shows_kind_field_and_source() -> None: alt = parse("JOHN SMITH MA").ambiguities[0].alternative - assert repr(alt) == "Reading('name' as 'family', 'declared')" + assert repr(alt) == "Reading('name' as 'family', 'declared', in en fil)" + # the files are left out where none holds the reading + assert repr(Reading("name", "family", "position")) == ( # type: ignore[arg-type] + "Reading('name' as 'family', 'position')") def test_parsedname_repr_lists_nonempty_fields_in_canonical_order() -> None: diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index d60f488e1..c4b949465 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -7,8 +7,8 @@ from nameparser import FAMILY_FIRST, GIVEN_FIRST, Policy, parse from nameparser._types import ( - STABLE_TAGS, Ambiguity, AmbiguityKind, ParsedName, Role, Segmentation, - Span, Token, + STABLE_TAGS, Ambiguity, AmbiguityKind, ParsedName, Reading, Role, + Segmentation, Span, Token, ) @@ -467,8 +467,9 @@ def test_types_pickle_round_trip() -> None: assert pickle.loads(pickle.dumps(seg)) == seg # Reading (#752 phase 2m) carries the same guarded pair from nameparser import Reading - reading = Reading("name", "family", "declared") # type: ignore[arg-type] + reading = Reading("name", "family", "declared", ("en",)) # type: ignore[arg-type] assert pickle.loads(pickle.dumps(reading)) == reading + assert pickle.loads(pickle.dumps(reading)).languages == ("en",) def test_span_add_is_blocked() -> None: @@ -520,6 +521,19 @@ def test_reading_coerces_its_strings_and_refuses_unknown_values() -> None: Reading("name", "family", "guess") # type: ignore[arg-type] +def test_reading_languages_are_a_tuple_of_tags_and_are_not_compared() -> None: + from nameparser import Reading + bare = Reading("particle", "family", "vocabulary") # type: ignore[arg-type] + held = Reading("particle", "family", "vocabulary", ["es", "fr"]) # type: ignore[arg-type] + assert bare.languages == () + assert held.languages == ("es", "fr") + assert held == bare and hash(held) == hash(bare) + with pytest.raises(TypeError, match=r"got the str 'es'; write \('es',\)"): + Reading("particle", "family", "vocabulary", "es") # type: ignore[arg-type] + with pytest.raises(TypeError, match="as str, got 1"): + Reading("particle", "family", "vocabulary", ("es", 1)) # type: ignore[arg-type] + + def test_ambiguity_alternative_is_a_reading_and_is_not_compared() -> None: from nameparser import Reading tok = Token("MA", Span(11, 13), Role.SUFFIX) @@ -543,6 +557,11 @@ def _token_state() -> dict[str, object]: ).__getstate__()) +def _reading_state() -> dict[str, object]: + return dict(Reading("name", "family", "declared", # type: ignore[arg-type] + ("en",)).__getstate__()) + + def _without(state: dict[str, object], *names: str) -> dict[str, object]: return {k: v for k, v in state.items() if k not in names} @@ -613,6 +632,15 @@ def _renamed(state: dict[str, object], old: str, "readings"), "unexpected fields: readings", id="renamed-reading"), + # Reading.languages (3.0) likewise + pytest.param(Reading, lambda: _without(_reading_state(), "languages", + "source"), + "missing fields: source;", + id="missing-shipped-field-beside-missing-languages"), + pytest.param(Reading, lambda: _renamed(_reading_state(), "languages", + "language"), + "unexpected fields: language", + id="renamed-languages"), ] @@ -673,6 +701,28 @@ def test_a_parsedname_pickled_by_2_8_0_loads() -> None: assert [t.reading for t in again.tokens] == [t.reading for t in now.tokens] +def test_a_reading_pickled_by_2_8_0_loads_without_languages() -> None: + """tests/v2/data/parsedname_v28_alternative.pickle is `parse("JOHN + SMITH MA")` pickled (protocol 4) by the released 2.8.0 wheel, its + Ambiguity's alternative a Reading written before + `Reading.languages` existed. Made 2026-10-29 the way the fixture + above was, version and module path asserted. pickle.load is safe + here: a repo-controlled fixture.""" + import pickle + with open(Path(__file__).parent / "data" + / "parsedname_v28_alternative.pickle", "rb") as f: + loaded = pickle.load(f) + now = parse("JOHN SMITH MA") + (old,), (new,) = loaded.ambiguities, now.ambiguities + assert old.alternative is not None and new.alternative is not None + assert old.alternative.languages == () + assert new.alternative.languages == ("en", "fil") + assert old.alternative == new.alternative + again = pickle.loads(pickle.dumps(now)) + assert [t.reading.languages for t in again.tokens if t.reading] == [ + t.reading.languages for t in now.tokens if t.reading] + + def test_a_parsedname_pickled_by_2_6_0_loads() -> None: """tests/v2/data/parsedname_v26.pickle is `parse("JOHN SMITH MA")` pickled (protocol 4) by the released 2.6.0 wheel, its Ambiguity From 61eee3cd2df3f30b49ea173e757ad8e02845577f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 29 Oct 2026 14:24:09 -0700 Subject: [PATCH 3/7] Answer review of Reading.languages: sorted unions, the split partner, the docs The code review found a nickname between a split credential's words joining the wrong neighbour ('Ph. "Bob" D.'), and unions over a word's parts or a report's words left unsorted against the documented order; both are fixed, the second word of such a split now taking its first word's reading across the nickname. The alternative's union is a loop, not a generator, so a report pays no resumed frame per word. The design-docs review found the RFC and a Standing line still saying the field was unfilled, an overbroad claim about which parses load the index (any word the parse's own Lexicon lists does, filed or not), and in-place rewrites of earlier dated entries, which are reverted in favour of a SUPERSEDES line and a head marker on decisions/A1. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- docs/design/decisions/A1-ambiguity-report.md | 4 +- docs/design/decisions/vocabulary-model.md | 18 +++++---- docs/design/vocabulary-model-rfc.md | 12 +++--- nameparser/_pipeline/AGENTS.md | 2 +- nameparser/_pipeline/_assemble.py | 41 +++++++++++++++----- nameparser/_types.py | 11 +++--- tests/v2/pipeline/test_assemble.py | 13 +++++++ 8 files changed, 70 insertions(+), 33 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8bd66e907..339251f9d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -205,7 +205,7 @@ Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely o ### Vocabulary (`nameparser/vocab/`) and the configuration layer (`nameparser/config/`) -**The default vocabulary is written per word, in TOML, and compiled.** `nameparser/vocab/.toml` holds one file per BCP 47 language tag, each `[[word]]` entry a `text` and its readings keyed by kind. What a file means, how an entry is built, the kinds and properties with the set each compiles into, exclusions, bearer counts and the steps for adding a word are in `nameparser/vocab/README.md`, the reference for anyone editing the TOML; `tests/v2/test_vocab.py` holds its kind, property, derived-marker and reserved-name tables to the names `build.py` accepts and the sets it compiles, so read it rather than restating it here. The parser never reads the TOML: `tools/vocab/build.py` runs the build checks that replaced the modules' import-time asserts (its docstring numbers them), turns every reading into an index row with its file's tag, and writes the rows as the committed reading index `nameparser/config/_index.py` (#753, phase 3a) and, compiled, as the committed `nameparser/config/_compiled.py`, frozenset literals one entry per line; the build never reads `_index.py` back. The compiler is `nameparser/_reading_index.py`'s `compile_readings`, with the kind and property tables it reads (`CATEGORY`, `BEHAVIORAL`, `GATED_WORDS`), so the build and phase 3's choice of languages compile alike; the build keeps the checks and loads that module by path, as it loads `_lexicon.py`, and `tests/v2/test_reading_index.py` holds that the index compiled with no choice is `_compiled.py` exactly. `readings_of(text)` there is the run-time reader of the index, every reading of a word with its file's tag, loaded on first call; `languages_of(text, categories)` is the reader a parse calls, from assemble, for `Reading.languages` (3.0), so the first parse that reads a listed word loads the index. Each `config/` module re-exports its constants from `_compiled.py` (`TITLES = _compiled.TITLES`) under its autodoc docstring, so the public names and the docs did not move. `tests/v2/test_vocab.py` fails when `_compiled.py` is stale, and `tests/v2/test_reading_index.py` when the index is, each naming the refresh command, and `test_vocab.py` gives every build check a mutation. The model, the phasing and what was declined: decisions/vocabulary-model. +**The default vocabulary is written per word, in TOML, and compiled.** `nameparser/vocab/.toml` holds one file per BCP 47 language tag, each `[[word]]` entry a `text` and its readings keyed by kind. What a file means, how an entry is built, the kinds and properties with the set each compiles into, exclusions, bearer counts and the steps for adding a word are in `nameparser/vocab/README.md`, the reference for anyone editing the TOML; `tests/v2/test_vocab.py` holds its kind, property, derived-marker and reserved-name tables to the names `build.py` accepts and the sets it compiles, so read it rather than restating it here. The parser never reads the TOML: `tools/vocab/build.py` runs the build checks that replaced the modules' import-time asserts (its docstring numbers them), turns every reading into an index row with its file's tag, and writes the rows as the committed reading index `nameparser/config/_index.py` (#753, phase 3a) and, compiled, as the committed `nameparser/config/_compiled.py`, frozenset literals one entry per line; the build never reads `_index.py` back. The compiler is `nameparser/_reading_index.py`'s `compile_readings`, with the kind and property tables it reads (`CATEGORY`, `BEHAVIORAL`, `GATED_WORDS`), so the build and phase 3's choice of languages compile alike; the build keeps the checks and loads that module by path, as it loads `_lexicon.py`, and `tests/v2/test_reading_index.py` holds that the index compiled with no choice is `_compiled.py` exactly. `readings_of(text)` there is the run-time reader of the index, every reading of a word with its file's tag, loaded on first call; `languages_of(text, categories)` is the reader a parse calls, from assemble, for `Reading.languages` (3.0), so the first parse that reads a word from its `Lexicon`'s vocabulary loads the index. Each `config/` module re-exports its constants from `_compiled.py` (`TITLES = _compiled.TITLES`) under its autodoc docstring, so the public names and the docs did not move. `tests/v2/test_vocab.py` fails when `_compiled.py` is stale, and `tests/v2/test_reading_index.py` when the index is, each naming the refresh command, and `test_vocab.py` gives every build check a mutation. The model, the phasing and what was declined: decisions/vocabulary-model. The `config/` SETS are frozen since 2.2 (#293): there is no `.add()`/`.remove()` on any of them, so a default word list is changed at runtime by configuring a `Lexicon`, never by editing the constant, and the shipped default is changed by editing the TOML and rebuilding. **Neither dict was frozen, and `CAPITALIZATION_EXCEPTIONS` is not covered by anything else either.** `REGEXES` is a compiled-pattern table rather than vocabulary and was never in #293's scope; `CAPITALIZATION_EXCEPTIONS` is vocabulary-shaped and is a decided, in-scope exemption. So the hazard the freeze closes is still live for it: `CAPITALIZATION_EXCEPTIONS['dclinpsy'] = 'DClinPsy'` after the first parse misses the cached `Lexicon.default()`. Same advice — configure the object (`dataclasses.replace(lexicon, capitalization_exceptions=...)`). `tests/v2/test_contracts.py::test_every_vocabulary_constant_is_frozen` names both dicts as explicit exemptions rather than letting its `isinstance` filter drop them. diff --git a/docs/design/decisions/A1-ambiguity-report.md b/docs/design/decisions/A1-ambiguity-report.md index bd4b47541..b55370d9d 100644 --- a/docs/design/decisions/A1-ambiguity-report.md +++ b/docs/design/decisions/A1-ambiguity-report.md @@ -30,12 +30,12 @@ Open: none in this file. - 2026-10-21 #793 — AMENDS #626's list of reports worded where the field is settled, for the comma's credential reading. The listed-member report after a family comma said "read as a credential" of a word the listing form binds as a title (`John, Si.`, `John, Ma.`), so that field was not settled by the wording alone. It now names the role the comma stage binds, "a title" or "a credential", worded at the emitter rather than through `field_tail`, since nothing later moves a word of the part the binding empties. The measurement is decisions/suffix-acronym-collisions' 2026-10-21 (#793) entry. -- 2026-10-26 #752 (phase 2m) — A REPORT CARRIES THE READING IT DID NOT TAKE. decisions/vocabulary-model's 2026-10-23 #752 entry planned it: an `Ambiguity` carries the reading the parse did not take, and a word the credential's run takes by position alone is told from a word with a declared name reading by that reading's source, so it needs no new ambiguity kind. Derek decided on 2026-10-26 to ship it in 2.7 and to hold the reading taken, `Token.reading`, for 3.0. +- 2026-10-26 #752 (phase 2m) (its DECLINED `languages` superseded by 2026-10-29 #754, 4a's follow-up, decisions/vocabulary-model) — A REPORT CARRIES THE READING IT DID NOT TAKE. decisions/vocabulary-model's 2026-10-23 #752 entry planned it: an `Ambiguity` carries the reading the parse did not take, and a word the credential's run takes by position alone is told from a word with a declared name reading by that reading's source, so it needs no new ambiguity kind. Derek decided on 2026-10-26 to ship it in 2.7 and to hold the reading taken, `Token.reading`, for 3.0. DECIDED: `Ambiguity.alternative: Reading | None`, declared `compare=False`, so two reports and two names compare as before and `ParsedName.comparison_key` is untouched. `Reading` is a frozen, slotted, picklable value of three fields, exported with its two enums: `kind` (`ReadingKind`: credential, generation, agnomen, post-nominal, title, particle, conjunction, initial, name), `role`, and `source` (`ReadingSource`). Its FIELD is the one the word would land in had the call gone the other way with every other word of the name read as the parse read it, which is what a caller moving one word needs. A non-name reading names its field by what it is: the suffix for a credential, a generation, an agnomen or another post-nominal, the title for a title, the family for a particle (rules/P2). A name reading takes the field the positional read gives one more name word in its place: the last name word's under the order that read the part (`JOHN SMITH MA` gives `family`, under FAMILY_FIRST `middle`), the given part's given or middle name after a family comma, the family before one, the maiden name in a marker's clause, and the given name of the listing form where the comma's decision bound the part as a credential run (`John Smith Jr., AD` gives `given`). Where the name piece in front would take the word in, it lands where that piece lands, named at assemble from the piece's final role: a particle chain, which runs to the end of the part's name words (`Juan de la Vega VI` under FAMILY_FIRST gives `given`, the chain's field), a lone particle not opening the part, which stood unchained only for want of a word behind it, and a bound given name opening the part where a family comma or a given-name title licenses its join (`Rahman, Abdul MB ChB` gives `given`, `Sir abdul VI` too); a particle P6 attached is not trailing once the word stands behind it, so it takes nothing in, and a bound join is no chain though `abu` is a particle too (`Smith, Abu Mary VI` gives `middle`). A never-given particle opening the part takes the word into the family by rules/P1's fold, in every order and part (`Dr. mc VI`, `Smith, Dr. de VI` give `family`). O5's lone word names the other primary field from its final role, after any move, and `middle_as_family` folds a middle into the family as it folds the parse's. DECIDED: four sources, where the RFC's "Readings on the parse" named three. `vocabulary` is a listed word in the reading's category, `declared` a word the vocabulary also declares a name (the derived markers of decisions/vocabulary-model's 2026-10-25 #752 entry, and a given name written short), `shape` a roman numeral, dotted letters or a one-letter initial, and `position` an ordinary word nothing lists, a name only by where it stands: `John Smith PhD Jones` gives `position` for `Jones` where `JOHN SMITH MA` gives `declared` for `MA`, which is the case the plan needed the source for. A shape-admitted member of the ambiguous class carries the class's tag beside the shape tag, so the shape tag decides that it declares no name. DECIDED: every emitter of every fork kind fills it at its decision site (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE). Two stages run before the field exists, classify's connective-or-initial letter and group's particle chain, and defer it (`_state.Deferred.JOINED` and `UNCHAINED`) for assign to name off the positions it reads: the connective's join takes in the name word on each side, so where those stand in two pieces the joined unit takes the place of the first (`JOSE E MARIA SANTOS` gives `given`), and where both are already inside the letter's own piece the join changes nothing the piece did not hold and the letter lands where its piece lands, named at assemble (`MARIA DA SILVA E SOUZA`, a case row, gives `family`); a join that would leave the name one word is none, rules/P3 keeping a single-letter connective in a three-word name a name word, so there too the letter keeps its own field (`JOHN E SMITH` gives `middle`), where a family comma's surname is the word that lets the given part join (`SMITH, JOHN E MARY` gives `given`); the unchained particle stands as one more unit in front of its piece. One assign cannot place stays unnamed and surfaces as no alternative, in two shapes on 2026-10-26: a connective letter opening its part, which has no word before it to join, so no field could hold the reading (`E SMITH JONES`, a case row since; `e j smith` and `e and e` in the committed corpora), and in the case sweep the chain's report in `Freiherr von Berg Dr. and Ed. Prof.`, under each order, whose chained piece ends in a title a connective's join made. The first was found by the design-docs review; rules/A1 names it. The three kinds that report a shape (comma-structure, unbalanced-delimiter, segmentation) carry none; `order` is reserved and emitted nowhere. ACCEPTED: a listed suffix word that is neither a generation by its tag nor an agnomen reads as `post-nominal`, so `jr` and `esq` alike, because telling them apart needs the per-word kind the reading index below would carry; only H4's input of nothing but post-nominals reaches it (`Rinpoche`). And the field holds every other word fixed, so it is not the field a re-parse with the word's reading removed would give wherever that re-parse reads something else differently: on 2026-10-26, 14 of the sweep's (row, order) pairs differ from a plain stand-in word that way. In 11 the stand-in moves a neighbour's reading (`John Smith VI Jones`, where a plain word for `VI` frees `Jones` from the run); in 3 it moves its own, `Andrew Perkins (X.Y.Z.)`, whose dotted content rules/S1 reads as if written bare, so its name reading is the bare word's, `family`, where a plain word in the brackets stays a nickname. A further 296 reports (299 once the PR review's `MARIA DA SILVA E SOUZA` row joined, 302 with #903's `Nasreen Sultana, MBBS`) cannot be checked that way at all, since the stand-in lands in the taken field: their decision was the position's. - DECLINED (for 2.7; carried out in 3.0, decisions/vocabulary-model's 2026-10-29 #754 entry on `Reading.languages`): the reading's `languages`, the files holding it. It needs a per-word reading index compiled into `_compiled.py`, the first parse-time use of the TOML's descriptive data. Measured 2026-10-26 on Python 3.11 on Derek's machine with bytecode cached, as an installed wheel has it: an index of the 1,790 words that hold a reading, 1,975 readings by word and kind, each with its files, added as a dict literal at the end of `_compiled.py`, costs about 0.35 ms of an 18.2 ms `import nameparser` (the median of 30 fresh processes, twice each, under a `PYTHONPYCACHEPREFIX`), about 2%, and a lookup costs each report a `_normalize` call. RECIPE for the index: for each `nameparser/vocab/*.toml` file and each `[[word]]` table, every table-valued key but `excluded` is a reading; group them by `text`, then kind, collecting the file stems. The cost is small; the reason is that the index describes the default vocabulary, not the `Lexicon` a parse used, so a word a caller or a pack adds would report no language and a word one removes would still report its file. It waits for 3.0 with `Token.reading`, which needs the same index. DECLINED too: naming a name reading's field by parsing again with the reading removed, which is exact and costs a parse per report, quadratic over the scaling shapes that report once per unit. + DECLINED: the reading's `languages`, the files holding it. It needs a per-word reading index compiled into `_compiled.py`, the first parse-time use of the TOML's descriptive data. Measured 2026-10-26 on Python 3.11 on Derek's machine with bytecode cached, as an installed wheel has it: an index of the 1,790 words that hold a reading, 1,975 readings by word and kind, each with its files, added as a dict literal at the end of `_compiled.py`, costs about 0.35 ms of an 18.2 ms `import nameparser` (the median of 30 fresh processes, twice each, under a `PYTHONPYCACHEPREFIX`), about 2%, and a lookup costs each report a `_normalize` call. RECIPE for the index: for each `nameparser/vocab/*.toml` file and each `[[word]]` table, every table-valued key but `excluded` is a reading; group them by `text`, then kind, collecting the file stems. The cost is small; the reason is that the index describes the default vocabulary, not the `Lexicon` a parse used, so a word a caller or a pack adds would report no language and a word one removes would still report its file. It waits for 3.0 with `Token.reading`, which needs the same index. DECLINED too: naming a name reading's field by parsing again with the reading removed, which is exact and costs a parse per report, quadratic over the scaling shapes that report once per unit. DECIDED: the pickle guard widened. `_types._guarded_setstate` refused any pickle whose field set differed from the class's, so the new field would have broken every `ParsedName` pickled with a report. A field added after its type shipped now carries `metadata={ADDED_IN: "2.7"}` and a default, and a pickle missing it loads with the default; a missing field without the marker (with a default or not: `Token.tags`, `ParsedName.ambiguities`), an unknown field and a renamed one are still refused. MEASURED 2026-10-26: each of the seven refusal rows in tests/v2/test_types.py raises under the pre-2.7 guard as under this one, and the admitted row and a `ParsedName` pickled by the 2.6.0 wheel (committed as tests/v2/data/parsedname_v26.pickle) raise under the pre-2.7 guard and load under this one; pickles from the 2.0.0 and 2.3.0 wheels load too. DECIDED: the snapshot records the alternatives, one string per report that carries one, in report order (`GIVEN_OR_FAMILY: name as family (position)`), so a later change moving one shows in the snapshot's diff, and losing one of two identical ones does too (`DOE, JOHN MA JD` holds two; the first draft recorded a set, which the PR review found hiding that). A snapshot's first line names the keys its recording tree could record (`snapshot.HEADER`), and a comparison leaves out a key either side did not record, read off that header rather than off whether a row holds one, which a 2.7 snapshot of names no report reaches would have failed; a snapshot without the header, a tag's or a local corpus's, records no alternatives (`snapshot.RECORDED_LATER`). `compare.py --baseline` against a release before 2.7 leaves them out too, its worker saying the wheel has no `Ambiguity.alternative`: on 2026-10-26 `--baseline 2.6.0` reported 1,088 entries moved with them compared, 867 of them only the key appearing, and 221 without, the role and ambiguity moves alone; `--since v2.6.0` reported 357 before and after. RECIPE for the measurements below: `uv run python tools/differential/snapshot.py`, `uv run python tools/differential/cldr_agreement.py`, and for each local corpus `snapshot.py --local --base ` and `local_agreement.py psc-individuals --base `. MISSED, then caught in review: the first draft named four fields wrong, each found by the code review's stand-in oracle on shapes no case row held. A credential or numeral with no name word before it counted the name word the carve-out makes as none (`PhD VI` gave `given`, where `family`). A numeral behind a particle P6 then attached took the particle's attached field (`Smith, John van VI` gave `family`, where `middle`), its test of P6's tag asked before P6 wrote it. A bound given name took a numeral behind it with no family word left for its join (`abdul VI` gave `given`, where `family`). And P6's own reports named the particle's field without asking whether the chain in front would take a name word in (`Juan de la Vega van` under FAMILY_FIRST gave `middle`, where `given`). The second review, of the fix commit, found two of those fixes narrowing elsewhere: the chain test the P6 reports now asked took a bound join for a chain (`Smith, Abu Mary van` gave `given`, where `middle`), and the bound join's licence was granted to a family comma alone, not to a given-name title (`Sir abdul VI` gave `family`, where `given`); it also found the P1 fold unmodelled (`Dr. mc VI` under FAMILY_FIRST gave `given`, where `family`). The PR's own review then found three more, and a review of its fixes a fourth. `capitalized()` rebuilt every report without its alternative, unseen by any test because the field is not compared, so the rebuilt name still equalled the original. A connective inside a joined piece got none, the join assumed to merge two pieces: 1,651 of the 4,967 connective reports in the tse-donors local corpus on 2026-10-26, 1 after the fix, the opening letter's. And the given-name title licence for the bound join was asked of every title token rather than of the leading run group's join asks through `_lexicon._run_addresses_by_given` (`abdul VI Sir.` gave `given`, where `family`; `Sir Dr. abdul VI` under FAMILY_FIRST, the run's last word not addressing by the given name, likewise). A review of those fixes found the connective's field ignoring P3's three-word carve-out, inherited from the first draft (`JOHN E SMITH` gave `given`, where its twin `John e Smith` reads middle 'e'; under FAMILY_FIRST `family`, where `given`). The shapes are rows of `_REVIEW_SHAPES` in tests/v2/test_cases.py, which fails 34 of its then 48 parses with the first round's fixes reverted, 14 with the second's, and 3 of its 57 with the third's licence fix; `test_an_alternative_has_its_recorded_kind_role_and_source` holds the reports the stand-in oracle cannot check (a credential run's, whose stand-in the same run reads, and a one-case connective's, whose stand-in is an initial too), one row per branch choosing a kind or a source, and the `middle_as_family` fold; and `test_capitalized_keeps_each_reports_alternative` compares the alternatives themselves. diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index c7622b78c..3aff947c3 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -19,7 +19,8 @@ The vocabulary was a set of base sets plus marker subsets (`particles` and `part - Title-phrase modifiers count as titles only beside a head title, at the head of the name only and never in a comma part (`Wing Lau` → given 'Wing', `John Smith, Vice` → title 'Vice'), save a modifier that may be a given name, which the head's test reads in every part (`Lau, Wing Kai` → given 'Wing'), rules/H9 — 2026-10-15/16; 2026-10-19 #750, title kinds; 2026-10-23 #752; 2026-10-25 #752, phase 2i; 2026-10-26 #752, phase 2j, `wing`. - Name readings are hand-declared, and only on a word some language file claims as vocabulary or refuses, so the vocabulary carries no surname list (the segmentation lists, Korean and the ZH pack's, are outside the per-word model) — 2026-10-15/16; 2026-10-19. - Bearer counts are structured provenance (source and count), never used to compute a reading — 2026-10-15/16. -- Phase 2 is to put vocabulary, declared name and shape readings in one pool, derive `may_be_given` and `may_be_name` from declared name readings (a count-free one carrying a `note`, `ld` left ungated, `may_be_initial` kept), and decide every clash by one gate of four steps, floor, form, structure and count, asked at the sites that decide today; the reading taken is to surface as `Token.reading` in 3.0 — 2026-10-23 #752. The reading not taken surfaces as `Ambiguity.alternative` since 2.7, without the vocabulary's languages (decisions/A1) — 2026-10-26 #752, phase 2m. +- Phase 2 is to put vocabulary, declared name and shape readings in one pool, derive `may_be_given` and `may_be_name` from declared name readings (a count-free one carrying a `note`, `ld` left ungated, `may_be_initial` kept), and decide every clash by one gate of four steps, floor, form, structure and count, asked at the sites that decide today; the reading taken is to surface as `Token.reading` in 3.0 — 2026-10-23 #752. The reading not taken surfaces as `Ambiguity.alternative` since 2.7 (decisions/A1) — 2026-10-26 #752, phase 2m. +- Every reading the parse took or a report names from the vocabulary names the shipped files holding it, `Reading.languages`, looked up at assemble in the reading index by the word and the kind; the default's files whatever `Lexicon` the parse used, and none for a shape, position, or a word no file holds in that category — 2026-10-29 #754, 4a's follow-up. - Every numeral rule asks one predicate, `_vocab.reads_as_generation`, the vocabulary's own numerals read inside it; a numeral of two or more letters is the generation wherever the trailing run holds it, while a one-letter one before a credential stays the name word decisions/S10's #721 entry made it — 2026-10-25 #752, phase 2f. - A credential written split across a space is one word, a member of the ambiguous class that the count decides, and the Ph./D. pair starts S8's run from where it stands (decisions/S12) — 2026-10-25 #752, phase 2e. - A listed credential that is also a name, written in capitals in a name of mixed case, is settled as the credential by its writing: it reports nothing and starts the credential's run where no name word stands behind it with no family comma (2026-10-26, Derek), and keeps its other readings, the clash settled at the sites that choose a reading rather than by retagging the word (decisions/S6) — 2026-10-25 #752, phase 2g, #785. @@ -275,22 +276,23 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) - **2026-10-28 #754, the `Lexicon` surface (Derek) — A CALLER CONFIGURES VOCABULARY BY READING, AND THE COMPILED SETS BECOME INTERNAL.** Derek decided in chat on 2026-10-28, choosing it over keeping a public field per compiled set and over keeping 2.0.0's eleven while hiding the eleven added since. A caller states what a word may be in the language files' terms (a `given` reading on `guru`, no credential reading on `dc`) and `Lexicon` derives the sets and gates as the build does. The reasons: the set mechanics are hard to keep straight even for the maintainer, a reading is what a caller can know about a word, and it is the form a pack contribution takes. Today the same configuration needs rules/H9's set algebra, `Lexicon.default().remove(titles={'guru'}).add(titles_ambiguous={'guru'})` (`Guru Prasad Sharma` → given `Guru`) and `remove(suffix_acronyms={'dc'})` (`Ram Bahadur DC` → family `DC`), while `add(suffix_acronyms_ambiguous={'dc'})` leaves suffix `DC` (2026-10-28, run on this branch's base). A caller's own removal may settle a clash: decision 15 binds the default and the packs, not a caller's data. `dataclasses.fields(Lexicon)` gives 22 public fields on 2026-10-28, eleven since 2.0.0's eleven; the RFC's Phase 4 "Left open" holds the weighing. A break of the frozen `Lexicon`, recorded in decisions/3-0-reevaluations. Open: the method names, and whether a caller's own language file can join the pool (phase 3's item 8 defers third-party files). The 2.x warning the freeze asks for was waived on 2026-10-29, 2.8 being the last 2.x feature release (decisions/3-0-reevaluations, THE LEXICON WARNING). - **2026-10-29 #753, phase 3a — THE READING INDEX AND THE SHARED COMPILER LAND, AND A NAME READING MAY BE WRITTEN `default = false`; NO PARSE MOVES.** Carries out items 3 and 5 of the 2026-10-27 #753 entry, and 3a of its item 11. - - **What landed.** tools/vocab/build.py writes a second generated module, nameparser/config/_index.py, beside _compiled.py: for each word, one row per reading in every file, on by default or not, `(tag, kind, properties)`, the properties those the compiler reads (each category's behavioral ones, and `same_as` on a `given` reading); counts, notes, exclusions and the stated `may_be_given` stay out, being checked at build time and compiled to nothing. The compiler moved into nameparser/_reading_index.py (`compile_readings`, with `CATEGORY`, `BEHAVIORAL` and `GATED_WORDS`), which the build loads by path as it loads _lexicon.py; the build keeps the checks and compiles _compiled.py from the index rows, so the module the parser reads is the index compiled. tests/v2/test_reading_index.py holds the no-choice invariant: `compile_readings(index())`, rendered as the build renders it, is _compiled.py byte for byte. The run-time reader is `readings_of(text)`, every `IndexReading` (text, tag, kind, props) of a word, for `Reading.languages`, which 4a (#942) left unfilled and its follow-up fills (the 2026-10-29 #754 entry on `Reading.languages` below, which reads the tags through `languages_of` beside it); `IndexReading.kind` is the vocabulary's kind (`credential`, `preposition`), and mapping it to a `ReadingKind` is that follow-up's. + - **What landed.** tools/vocab/build.py writes a second generated module, nameparser/config/_index.py, beside _compiled.py: for each word, one row per reading in every file, on by default or not, `(tag, kind, properties)`, the properties those the compiler reads (each category's behavioral ones, and `same_as` on a `given` reading); counts, notes, exclusions and the stated `may_be_given` stay out, being checked at build time and compiled to nothing. The compiler moved into nameparser/_reading_index.py (`compile_readings`, with `CATEGORY`, `BEHAVIORAL` and `GATED_WORDS`), which the build loads by path as it loads _lexicon.py; the build keeps the checks and compiles _compiled.py from the index rows, so the module the parser reads is the index compiled. tests/v2/test_reading_index.py holds the no-choice invariant: `compile_readings(index())`, rendered as the build renders it, is _compiled.py byte for byte. The run-time reader is `readings_of(text)`, every `IndexReading` (text, tag, kind, props) of a word, for `Reading.languages`, which 4a (#942) adds but leaves unfilled and a follow-up after 3a and 4a both merge fills; `IndexReading.kind` is the vocabulary's kind (`credential`, `preposition`), and mapping it to a `ReadingKind` is that follow-up's. - **A name reading written `default = false` derives no marker in the default** (item 3, the half before a pack turns it on): the word's other readings compile as they would without it. So it counts for none of the checks guarding the default's compile: an agnomen on by default needs a name reading on by default (check 11, else it reads ungated), and a bound given name that is also a particle, which needs a `given` reading on by default (else Lexicon raises at import), is refused by check 11's particle half below before check 9 reads it. A name reading off beside a suffix word kind no gate reads is still refused, gating nothing when on either. - **Left open: a particle whose word's only `given` readings are off is refused.** Check 11 asks such a particle to state `may_be_given = false` exactly where its word has no `given` reading. The default compiles it never-given, a pack turning the given reading on compiles it possibly given, and which of the two the stated property describes is a choice no 3.0 pack needs yet (`nl`'s prepositions wait with strict). The build refuses the shape, naming both ways out, rather than pick a side; the first pack that needs it decides, with its measurement. - **The index is written grouped by word, and an `IndexReading` is built only for a word looked up.** On a flat draft of this change, one row per line, grouping the rows at run time took about 0.56 ms; building every row's object takes about 1.4 ms (`index()`, which only a compile over the whole pool calls); each against the import's own cost, so the build writes one line per word and the lookup table is a single `dict()` call. A change to a word's readings is still a one-line diff. - - **The index loads on the first `readings_of` call, not inside `Lexicon.default()` (decided 2026-10-29, Derek).** SUPERSEDES: item 5's "loaded for every caller with `Lexicon.default()`". Proposed by 3a and chosen over loading it with the default lexicon. What it buys depends on the state of `Reading.languages`. Until the follow-up that fills it (the 2026-10-29 #754 entry on `Reading.languages` below), no parse read the index, so no caller paid the import, parsing or not, where loading it with the default lexicon would have cost every parsing caller the import for nothing. Since that follow-up, the first parse to read a word from the vocabulary loads it, so every caller who parses such a name pays it once, about 1 ms on that parse (measured there), and a caller who imports nameparser without parsing, or parses only words no file lists, is spared; the import moved with the call, into assemble. + - **The index loads on the first `readings_of` call, not inside `Lexicon.default()` (decided 2026-10-29, Derek).** SUPERSEDES: item 5's "loaded for every caller with `Lexicon.default()`". Proposed by 3a and chosen over loading it with the default lexicon. What it buys depends on the state of `Reading.languages`. Today, and until the follow-up that fills it after 3a and 4a (#942) both merge, no parse reads the index, so no caller pays the import, parsing or not, where loading it with the default lexicon would cost every parsing caller the import for nothing. Once the follow-up fills `Reading.languages`, the first parse will load it, so every parsing caller will pay it once, as item 5 measured, and only a caller who imports nameparser without parsing is spared. Wherever the follow-up calls `readings_of`, the import moves with the call. - **Measured** 2026-10-29 on `9210f221` plus this change, Python 3.11.17, medians of fresh processes with bytecode cached as an installed wheel has it: importing the index takes 0.56 to 0.68 ms across three runs on a shared machine, about 3% of `import nameparser`'s 18 to 24 ms in the same runs, above the 0.45 ms item 5 measured on a flat draft of 2,140 rows; the first lookup after it, 0.07 ms; compiling the whole index, 2.1 ms; uncached, the import takes about 13 to 18 ms (13 after `import nameparser` in this session, 17 to 19 in review's runs, a figure no wheel install pays). The index holds 2,157 readings over 1,798 words, 106 of them in more than one file, and 3 readings written `default = false`; it is about 95 KB of source. Recompute: `uv run python tools/vocab/build.py`, then with `PYTHONDONTWRITEBYTECODE` unset, time `import nameparser.config._index` after `import nameparser` in fresh processes (the first run writes the bytecode), and count with `from nameparser.config._index import READINGS`. - Not touched: `Lexicon` and every other pickled type gain no field, so no pickle layout moves, and 3a forces no public name; the configure-by-reading methods stay open. -- **2026-10-29 #754, sub-phase 4a — `Token.reading` IS READ AT ASSEMBLE, FROM EACH TOKEN'S FINAL ROLE AND THE TAGS THE DECIDING STAGES LEFT, ITS SUFFIX AND NAME SOURCES BY THE PREDICATES THAT NAME `Ambiguity.alternative`.** It carries out item 2 of the 2026-10-28 #754 entry above, all but `Reading.languages`, which a follow-up adds and fills from 3a's reading index (`_reading_index.readings_of`, the 2026-10-29 #753 entry above), adding the field with its values because an empty one would say that no file holds the reading; the follow-up landed the same day (the entry below). +- **2026-10-29 #754, sub-phase 4a — `Token.reading` IS READ AT ASSEMBLE, FROM EACH TOKEN'S FINAL ROLE AND THE TAGS THE DECIDING STAGES LEFT, ITS SUFFIX AND NAME SOURCES BY THE PREDICATES THAT NAME `Ambiguity.alternative`.** It carries out item 2 of the 2026-10-28 #754 entry above, all but `Reading.languages`, which a follow-up adds and fills from 3a's reading index (`_reading_index.readings_of`, the 2026-10-29 #753 entry above), adding the field with its values because an empty one would say that no file holds the reading. 1. **Where it is filled.** A word's reading is decided in two halves today: its tags say which readings of the word the vocabulary or a shape offers and how its part stands (classify's vocabulary and shape tags, the `joined` mark group and post_rules write, post_rules' unjoined marks, which make `van` in `van Gogh` a name word where `Vincent van Gogh` keeps it a particle), and the stage that gives the word its role says which one was taken. Both halves are final only at assemble, since post_rules still moves words after assign (P1's fold, P6's attachment, the patronymic rotations, `middle_as_family`'s fold), so a reading written at an earlier site would need re-writing at every later move, the stale-mark class `_types._remarked` exists to recompute. Assemble reads the suffix and name readings through `_vocab.suffix_kind` and `_vocab.name_source`, the predicates `suffix_reading` and `name_reading` use for the reading NOT taken, so the two are named by one predicate there. The bound given name's `vocabulary` and the title, connective, particle and initial readings are assemble's own tests, kept out of the shared predicates so that no alternative moves (the first draft put the bound given name in `name_source` and moved `Smith, John Abu`'s from `declared`, which review caught; `_EXPECTED_ALTERNATIVES` holds it now), the initial by classify's `initial` tag where `name_reading` names one by its shape (a one-letter `é` in `John Smith é` reads `name`, the tag never written on it). Neutral, measured 2026-10-29: the committed snapshot is byte-identical, every recorded alternative included. 2. **What each role reads as.** A title: `title` from the vocabulary where a title tag stands, by its shape where rules/H2's period-abbreviation shape claimed it (`Insp.`, `Esq. Smith`), and by position where a title phrase took a word nothing lists (`Parliament` in `Member of Parliament`); its connective `conjunction`. A suffix: `suffix_kind`'s kind and source, save a word no suffix tag marks and the vocabulary does not list, which stands there by position: a word past a second comma (`Foo` in `John Smith, Jr., Foo`), or a continuation of the entry before it (`Jones` in `John Smith PhD Jones` and in `John Smith III Jones` reads `credential`, `position`), save a credential split across a space, whose second word takes its first word's reading whole (`Ph.` `D.`); a connective reads `conjunction` from the vocabulary, as in a title or a name part, whether it joins two entries (`and` in `John Smith, PhD and MD`) or trails them with nothing to join (`John Smith, Jr. and`), as `John Smith and` does on the name side. A name-role word: `initial` by its shape where classify tagged it one, `conjunction` or `particle` from the vocabulary where it joins, and otherwise `name`, `declared` where a derived marker says so, `vocabulary` for a word the vocabulary lists as a bound given name, whether rules/P5 joined it (`Abdul` in `Abdul Aziz Tariq`) or not (`Abdul` as the middle name in `John Abdul Smith`), save one that is also a particle (`abu`), which is P5's given name only where P5 joins it, in the given part and opening its segment past the titles (`Abu Bakar Smith`, `Smith, Abu Bakar`, `Dr. Abu Bakar Smith`), and otherwise a `particle` where it joins, the chain having taken it (`John Abu Smith`, `Smith, John Abu`, and the given part of `Salim Ali abu Bakar Hassan` under FAMILY_FIRST_GIVEN_LAST, where P5 joins nothing past the family it opens with), no tag telling the two joins apart, and for a family word the segmentation split off an unspaced name by `Lexicon.surnames` (`김` in `김민준`, where a spaced `김 민준` reads `position`, the order placing it), and `position` for the rest; a particle or connective with nothing to join reads as the name word it stands as (`van` in `van Gogh`, `name`, `declared`), and so does a particle in a nickname, which has no family to join (`"Van" Smith`). 3. **Not filled**, `None`: a token `ParsedName.replace()` builds, a token `Parser.revise()` harvests, whose role the caller forced, and a token loaded from a pickle written before 3.0 (`ADDED_IN = "3.0.0"`; `tests/v2/data/parsedname_v28.pickle`, made by the released 2.8.0 wheel, loads and equals a fresh parse). `capitalized()` carries each reading through. 4. **Cost**, measured 2026-10-29 with `tools/perf/call_count.py`: the reference row moved from 305 to 306 frames on Python 3.11 and from 286 to 287 on 3.15, one frame for its suffix `III`. Every `Reading` a token can take is built once at import (280 of them, ten kinds by seven roles by four sources) and shared, where a construction per token would cost two frames a word; an untagged word, the commonest, is read in C-level tests with no frame. `_CALL_BASELINE` (307 and 288) is not moved. 5. **Left open.** A listed suffix word that the compiled sets do not split by kind reads `post-nominal`, so `Jr`, `Sr` and `Esq.` read alike: the TOML gives `jr` a `generation` reading, but `SUFFIX_WORDS` holds both kinds and classify tags them alike, so only a vocabulary lookup by kind can tell them apart: 3a's reading index, a new `Lexicon` field being set aside (the 2026-10-28 `Lexicon` surface entry makes the compiled sets internal, and `Lexicon`'s own pickle guard refuses an added field). It is the predicate's own answer, which `Ambiguity.alternative` has given since 2.7 (the snapshot's one post-nominal alternative is `Rinpoche`'s), and a kind left unset would have meant a parsed token with no reading or a kind for "not yet known", so the reading taken agrees with it until 4b, whose `generation` view needs the split, reads a listed word's kind from 3a's index; `tests/v2/pipeline/test_assemble.py::test_a_suffix_word_reads_as_its_vocabulary_kind` holds `Jr` and `Sr` as strict expected failures until then. `ReadingKind.PATRONYMIC` exists and no reading takes it until 4c, whose rules write it. This amends the 2026-10-28 #754 entry's item 9: 4b needs 3a for that kind. -- **2026-10-29 #754, 4a's follow-up — `Reading.languages` IS FILLED AT ASSEMBLE FROM 3a's READING INDEX, FOR THE READING A TOKEN TOOK AND FOR A REPORT'S ALTERNATIVE; NO PARSE MOVES.** Completes 4a (the entry above) with the field item 2 of the 2026-10-28 #754 entry names, and reverses for 3.0 the decline of decisions/A1's 2026-10-26 entry. - 1. **What it holds.** The BCP 47 tags of the shipped files holding the reading taken, on by default, each once, in tag order, read through `_reading_index.languages_of(text, categories)`, which walks the index rows without building an `IndexReading`. A reading matches a file's rows by category, not by the vocabulary's finer kind: a `title`, `particle` or `conjunction` reading its category's rows, a `name` reading the `name` and `bound_given` rows, and every suffix kind the word's suffix rows, a word having one suffix kind (the build's check 4), so `Jr`, which reads `post-nominal` until 4b, names `en`, the file of its `generation` reading. Only a `vocabulary` or `declared` reading is looked up; a shape, position, an initial or a patronymic names no file. A reading written `default = false` names none (`St` in a caller's `suffix_words`, held off by `cs.toml`), and neither does a word a caller's own `Lexicon` adds, nor the Korean surnames that divide unspaced hangul, which `config/surnames.py` keeps outside the per-word files: `김` in `김민준` reads `name`, `vocabulary`, with no files. It names the default's files whatever `Lexicon` the parse used, the semantics decisions/A1's 2026-10-26 decline weighed against it: a file holds the reading or not, and which lexicon read it is the caller's own knowledge. Under a choice of languages (3b) a "live" reading becomes one the choice makes, and `languages_of` filtering on `on_by_default` is the place 3b changes. - 2. **How a word is looked up.** By the vocabulary's own form of the word (`_normalize`); a suffix also with every period removed, as a credential is matched (`M.A` in `John Smith (M.A)`); a credential split across a space whole, its first word joined to the second (`Ph.` `D.` as `phd`), the second taking the first's reading; and a word joining listed words with punctuation by its parts, in order, a suffix's parts divided by any mark but the period first (`M.D./Ph.D.`, `MD;`), then any word's by every mark (`Lt.Gov.`), never at a combining mark, which `\W` would count (`प्रोफे.डॉ.` names `hi` and `mr` through `डॉ`). An alternative names the files for any word its report names, in order. MEASURED 2026-10-29 on `bd02d68f` plus this change over the snapshot's 6,359 distinct names (20,804 tokens): 3,935 readings are `vocabulary` or `declared`, 3,778 of them name a file, and the 157 that name none are all Korean surnames split off by the list; by the normalized form and the suffix's period-free form alone, 265 would name none, the further 108 being the words of credentials split across a space, 40 by `split:credential` (`Ph.` `D.`) and 26 by `split:member` (`Pharm D`), and 42 words joining listed words. Recompute: parse every `name` of `tools/differential/snapshot.jsonl` and count the tokens whose `reading.source` is `vocabulary` or `declared` by whether `reading.languages` is empty; for the second figure, look each up with `languages_of` by its normalized and period-free forms only, and sort the misses by the split tags on the word and the word before it. +- **2026-10-29 #754, 4a's follow-up — `Reading.languages` IS FILLED AT ASSEMBLE FROM 3a's READING INDEX, FOR THE READING A TOKEN TOOK AND FOR A REPORT'S ALTERNATIVE; NO PARSE MOVES.** Completes 4a (the 2026-10-29 #754, sub-phase 4a entry) with the field item 2 of the 2026-10-28 #754 entry names. Until it, no parse read the index (the 2026-10-29 #753, phase 3a entry's load bullet); since it, the first parse that reads a word from the vocabulary of its own `Lexicon` loads it (item 4). + SUPERSEDES: decisions/A1's 2026-10-26 #752 (phase 2m) entry's decline of the reading's `languages`, for 3.0; its reason, that the index describes the default and not the `Lexicon` a parse used, is answered by item 1's semantics rather than removed. + 1. **What it holds.** The BCP 47 tags of the shipped files holding the reading taken, on by default, each once, sorted (a word joined from parts and an alternative over several words sorting the union), read through `_reading_index.languages_of(text, categories)`, which walks the index rows without building an `IndexReading`. A reading matches a file's rows by category, not by the vocabulary's finer kind: a `title`, `particle` or `conjunction` reading its category's rows, a `name` reading the `name` and `bound_given` rows, and every suffix kind the word's suffix rows, a word having one suffix kind (the build's check 4), so `Jr`, which reads `post-nominal` until 4b, names `en`, the file of its `generation` reading. Only a `vocabulary` or `declared` reading is looked up; a shape, position, an initial or a patronymic names no file. A reading written `default = false` names none (`St` in a caller's `suffix_words`, held off by `cs.toml`), and neither does a word a caller's own `Lexicon` adds that no shipped file holds in that category (`Zzq` as a title), while one a file does hold names it (`Dean` added to `titles` names `en`), nor the Korean surnames that divide unspaced hangul, which `config/surnames.py` keeps outside the per-word files: `김` in `김민준` reads `name`, `vocabulary`, with no files. It names the default's files whatever `Lexicon` the parse used, the semantics decisions/A1's 2026-10-26 decline weighed against it: a file holds the reading or not, and which lexicon read it is the caller's own knowledge. Under a choice of languages (3b) a "live" reading becomes one the choice makes, and `languages_of` filtering on `on_by_default` is the place 3b changes. + 2. **How a word is looked up.** By the vocabulary's own form of the word (`_normalize`); a suffix also with every period removed, as a credential is matched (`M.A` in `John Smith (M.A)`); a credential split across a space whole, its first word joined to the second (`Ph.` `D.` as `phd`), the second taking the first's reading; and a word joining listed words with punctuation by its parts, in order, a suffix's parts divided by any mark but the period first (`M.D./Ph.D.`, `MD;`), then any word's by every mark (`Lt.Gov.`), never at a combining mark, which `\W` would count (`प्रोफे.डॉ.` names `hi` and `mr` through `डॉ`). An alternative names the files for any word its report names. MEASURED 2026-10-29 on `bd02d68f` plus this change over the snapshot's 6,360 distinct names, the empty one among them (20,804 tokens): 3,935 readings are `vocabulary` or `declared`, 3,778 of them name a file, and the 157 that name none are all Korean surnames split off by the list; by the normalized form and the suffix's period-free form alone, 265 would name none: the 157, 42 words joining listed words, and 66 words of credentials split across a space, 40 by `split:credential` (`Ph.` `D.`) and 26 by `split:member` (`Pharm D`). Recompute: parse every `name` of `tools/differential/snapshot.jsonl` and count the tokens whose `reading.source` is `vocabulary` or `declared` by whether `reading.languages` is empty; for the second figure, look each up with `languages_of` by its normalized and period-free forms only, and sort the misses by the split tags on the word and the word before it. 3. **The field.** `Reading.languages: tuple[str, ...] = ()`, `compare=False`, so two readings compare as in 2.x and every recorded alternative is untouched, and `metadata={ADDED_IN: "3.0.0"}`, `Reading` having shipped in 2.7: a `Reading` pickled by 2.7 or 2.8 loads with `()` (`tests/v2/data/parsedname_v28_alternative.pickle`, made by the released 2.8.0 wheel), and a renamed field or a shipped one missing beside it is still refused (`_REFUSED_SKEWS`). The constructor takes any iterable of str and refuses a bare str and a non-str tag. The repr names the files where there are any (`Reading('particle' as 'family', 'vocabulary', in es fr it nl pt)`). The snapshot does not record readings, so nothing re-records; it is byte-identical. - 4. **Cost**, measured 2026-10-29, Python 3.11.17. Two memos keep the lookups off the frame count: `_languages`, an `lru_cache(maxsize=4096)` keyed on the word as written and the kind, and an `lru_cache(maxsize=1024)` over `Reading` for the readings that name files, each a pure function of the shipped files, bounded because the caller's text keys them, and C-level on a hit, so a word read before costs no frame. `tools/perf/call_count.py`'s reference row moved from 311.00 on `bd02d68f` to 311.40, all of it the first reference parse of the 50 (331 frames against 311: the index import and the four first lookups); after it each parse counts 311.0 on both trees. `_CALL_BASELINE` (307) is not moved and the serial benchmark passes. In fresh processes with bytecode cached, as an installed wheel has it, the first parse to read a listed word (`Dr. Juan de la Vega III`, after a `John Smith`) takes 1.20 to 1.22 ms against 0.22 ms on `bd02d68f`, the medians of 21 processes in each of three runs: about 1.0 ms once per process, the index import 3a measured at 0.56 to 0.68 ms plus the first lookups and readings. The steady parse, over 4,000 names half of them listed-word names, takes 78.6 to 79.0 µs against 78.3 to 79.4 µs on `bd02d68f`, within the runs' noise. Recompute: time the two parses in fresh processes and a `timeit` over the 4,000 names in each tree, with `PYTHONDONTWRITEBYTECODE` unset and the index's bytecode written. + 4. **Cost**, measured 2026-10-29, Python 3.11.17. Two memos keep the lookups off the frame count: `_languages`, an `lru_cache(maxsize=4096)` keyed on the word as written and the kind, and an `lru_cache(maxsize=1024)` over `Reading` for the readings that name files, each a pure function of the shipped files, bounded because the caller's text keys them, and C-level on a hit, so a word read before costs no frame. `tools/perf/call_count.py`'s reference row moved from 311.00 on `bd02d68f` to 311.40, all of it the first reference parse of the 50 (331 frames against 311: the index import and the four first lookups); after it each parse counts 311.0 on both trees. `_CALL_BASELINE` (307) is not moved and the serial benchmark passes. Which parse loads the index is decided by the parse's own `Lexicon`, not the files: any `vocabulary` or `declared` reading looks its word up, so `김민준` and a caller's own title (`Zzq John Smith`) load it though no file holds either, while `John Smith` does not. In fresh processes with bytecode cached, as an installed wheel has it, the first parse to read a listed word (`Dr. Juan de la Vega III`, after a `John Smith`) takes 1.20 to 1.22 ms against 0.22 ms on `bd02d68f`, the medians of 21 processes in each of three runs: about 1.0 ms once per process, the index import 3a measured at 0.56 to 0.68 ms plus the first lookups and readings. The steady parse, over 4,000 names half of them listed-word names, takes 78.6 to 79.0 µs against 78.3 to 79.4 µs on `bd02d68f`, within the runs' noise. Recompute: time the two parses in fresh processes and a `timeit` over the 4,000 names in each tree, with `PYTHONDONTWRITEBYTECODE` unset and the index's bytecode written. 5. **Not done here: a listed suffix word's kind.** The index lookup this change makes does not settle 4b's `Jr` and `Sr` (`tests/v2/pipeline/test_assemble.py`'s strict expected failures stand): the kind a token takes is named by `_vocab.suffix_kind`, which also names `Ambiguity.alternative`, so reading the kind from the index at assemble alone would split the two, and reading it inside `suffix_kind` moves the alternatives and puts the index on the decision path. 4b decides it. diff --git a/docs/design/vocabulary-model-rfc.md b/docs/design/vocabulary-model-rfc.md index 7df3d8e7e..432370140 100644 --- a/docs/design/vocabulary-model-rfc.md +++ b/docs/design/vocabulary-model-rfc.md @@ -134,7 +134,7 @@ Phase 2l carried this out as [rules/S13](rules/S13-bracketed-member-settled-by-i ### Readings on the parse -A caller should be able to see the reading behind a parse. **An `Ambiguity` carries the reading the parse did not take** (`Ambiguity.alternative`: its kind, the field it would have put the token in, and whether it came from the vocabulary, a declared name or a shape), and **a token carries the reading it did** (`Token.reading`), so a word whose reading forks nothing still says what it was read as, without a report nobody would act on (phase 4's 4a; a native spelling such as `do`'s Vietnamese `Đỗ` is not part of it). A word the credential's run takes by position alone is told from a word with a declared name reading by that source, so it needs no new ambiguity kind. `Ambiguity.alternative` shipped in 2.7 (decisions/A1, 2026-10-26 #752): a `Reading` of a kind, the field the word would land in with every other word read as the parse read it, and a source, which is `vocabulary`, `declared`, `shape` or `position`, the fourth being the reading of an ordinary word nothing lists, which is what a word the credential's run took by position alone has. The pickle guard was widened to accept a field added later. A `languages` field, naming the files that hold a reading, was weighed and left out: it needs the per-word reading index, which costs about 0.35 ms of an 18.2 ms import (decisions/A1) and could not describe a pack's or a caller's lexicon, so it waits with `Token.reading` for 3.0. Phase 4's 4a carried out `Token.reading` on 2026-10-29, the `languages` still waiting for phase 3's index (decisions/vocabulary-model's 2026-10-29 #754 entry). +A caller should be able to see the reading behind a parse. **An `Ambiguity` carries the reading the parse did not take** (`Ambiguity.alternative`: its kind, the field it would have put the token in, and whether it came from the vocabulary, a declared name or a shape), and **a token carries the reading it did** (`Token.reading`), so a word whose reading forks nothing still says what it was read as, without a report nobody would act on (phase 4's 4a; a native spelling such as `do`'s Vietnamese `Đỗ` is not part of it). A word the credential's run takes by position alone is told from a word with a declared name reading by that source, so it needs no new ambiguity kind. `Ambiguity.alternative` shipped in 2.7 (decisions/A1, 2026-10-26 #752): a `Reading` of a kind, the field the word would land in with every other word read as the parse read it, and a source, which is `vocabulary`, `declared`, `shape` or `position`, the fourth being the reading of an ordinary word nothing lists, which is what a word the credential's run took by position alone has. The pickle guard was widened to accept a field added later. A `languages` field, naming the files that hold a reading, was weighed and left out: it needs the per-word reading index, which costs about 0.35 ms of an 18.2 ms import (decisions/A1) and could not describe a pack's or a caller's lexicon, so it waits with `Token.reading` for 3.0. Phase 4's 4a carried out `Token.reading` on 2026-10-29 (decisions/vocabulary-model's 2026-10-29 #754, sub-phase 4a entry), and its follow-up `Reading.languages` the same day, from phase 3's index (the 2026-10-29 #754, 4a's follow-up entry). ### The phase 1 reserved names @@ -219,7 +219,7 @@ The 2 broken are pairs whose romanization opens with a listed title, which reads ### Where the provenance lives at runtime -The compiled module holds sets, and a set has forgotten which file each word came from. Phase 3 compiles a second module beside it (decided 2026-10-28): a **reading index**, one row per reading (its text, its file's tag, its kind and its behavioral properties), loaded once per process on its first lookup (`_reading_index.readings_of`) (decided 2026-10-29, replacing the 2026-10-28 decision to load it with `Lexicon.default()`). Phase 4's `Reading.languages` is filled from it at assemble (4a's follow-up, decisions/vocabulary-model's 2026-10-29 #754 entry on it), so the first parse in a process that reads a word from the vocabulary loads it: every caller who parses such a name pays it once, about 1 ms on that parse, and a caller who only imports nameparser, or parses only words no file lists, does not. Filling `reading.languages` on first access to that field instead was weighed and set aside: a `Reading` holds its kind, role and source but not its word, so a lazy field would have to carry the word as a hidden key on a frozen, slotted, pickled type, to save a cost the measurement below puts at about 2% of the import. The compiler moves from tools/vocab/build.py into the package so that the build and a pack share one implementation, and a test holds the invariant that compiling the index with no choice reproduces `nameparser/config/_compiled.py` exactly. The same index is what item 13 of decisions/vocabulary-model's 2026-10-23 #752 entry says `Token.reading` needs in 3.0; 3a built it (decisions/vocabulary-model, 2026-10-29), its import measured there at 0.56 to 0.68 ms. +The compiled module holds sets, and a set has forgotten which file each word came from. Phase 3 compiles a second module beside it (decided 2026-10-28): a **reading index**, one row per reading (its text, its file's tag, its kind and its behavioral properties), loaded once per process on its first lookup (`_reading_index.readings_of`) (decided 2026-10-29, replacing the 2026-10-28 decision to load it with `Lexicon.default()`). Phase 4's `Reading.languages` is filled from it at assemble (4a's follow-up, decisions/vocabulary-model's 2026-10-29 #754 entry on it), so the first parse in a process that reads a word from its `Lexicon`'s vocabulary loads it (a listed or declared word, a Korean surname dividing hangul or a caller's own word, whether or not a file holds it): every caller who parses such a name pays it once, about 1 ms on that parse, and a caller who only imports nameparser, or parses only words its `Lexicon` does not list, does not. Filling `reading.languages` on first access to that field instead was weighed and set aside: a `Reading` holds its kind, role and source but not its word, so a lazy field would have to carry the word as a hidden key on a frozen, slotted, pickled type, to save a cost the measurement below puts at about 2% of the import. The compiler moves from tools/vocab/build.py into the package so that the build and a pack share one implementation, and a test holds the invariant that compiling the index with no choice reproduces `nameparser/config/_compiled.py` exactly. The same index is what item 13 of decisions/vocabulary-model's 2026-10-23 #752 entry says `Token.reading` needs in 3.0; 3a built it (decisions/vocabulary-model, 2026-10-29), its import measured there at 0.56 to 0.68 ms. Measured 2026-10-27 on `cfe8bf92`: the pool holds 2,140 readings over 1,797 words, 106 of them in more than one file. An index of their behavioral properties is about 93 KB of generated source. Imported with its bytecode cached, as an installed wheel has it, it takes about 0.45 ms (median of nine fresh processes, 2026-10-28 on `c8357e74`, Python 3.11.17), against about 20 ms for `import nameparser` cached the same way; with no cache it takes about 11 ms, the figure this section gave on 2026-10-27, measured in a shell that sets `PYTHONDONTWRITEBYTECODE`. That load is paid once per process, at the first lookup. Compiling the pool takes about 9 ms, paid once per choice and only by a caller who makes one. Reading the TOML instead costs about 39 ms. Recompute: the build's `read_dir`, `load` and `compile_pool`, timed; for the import, write the index as a module, `py_compile` it, and time its import in fresh processes beside a cached `import nameparser`. @@ -272,7 +272,7 @@ Languages with one to three candidates (`tr` `bay`, `it` `sig`, `lt` `jaun`, `ms ## Phase 4: order, shape and the output model -Status: **the output model is agreed** (2026-10-28, #754), and sub-phase 4a has landed, `Reading.languages` with it (2026-10-29, decisions/vocabulary-model's two 2026-10-29 #754 entries). The record is decisions/vocabulary-model's 2026-10-28 #754 entry; the output-model and work-order subsections below are the design its items describe. The first two subsections, order and shape and the rules a pack can choose, are as written before and are amended only where the design reaches them. Phase 4 is 3.0 work. The core API (`parse`, `Parser`, `Lexicon`, `Policy`, `Locale`, `ParsedName`) is frozen for 3.0 except for additions (decisions/3-0-reevaluations, 2026-10-28 #925, THE CORE API), so every change below says whether it is an addition. +Status: **the output model is agreed** (2026-10-28, #754), and sub-phase 4a has landed, `Reading.languages` with it (2026-10-29, decisions/vocabulary-model's 2026-10-29 #754, sub-phase 4a and 4a's follow-up entries). The record is decisions/vocabulary-model's 2026-10-28 #754 entry; the output-model and work-order subsections below are the design its items describe. The first two subsections, order and shape and the rules a pack can choose, are as written before and are amended only where the design reaches them. Phase 4 is 3.0 work. The core API (`parse`, `Parser`, `Lexicon`, `Policy`, `Locale`, `ParsedName`) is frozen for 3.0 except for additions (decisions/3-0-reevaluations, 2026-10-28 #925, THE CORE API), so every change below says whether it is an addition. ### Order is not shape @@ -315,7 +315,7 @@ Phase 4 also decides what a parse returns, aligned with CLDR's person-name field What decides a view is the reading each token took, so `Token.reading` comes first; `surname2` alone is read from a tag O6 writes, a rule rather than a word deciding it: -- **`Token.reading`**: a `Token` field holding the `Reading` the parse took, the same type `Ambiguity.alternative` holds the reading not taken in (decisions/A1), `None` for a token `replace()` or `revise()` builds. Its `kind` says what the word was read as, its `source` why, and a `languages` field, added to `Reading`, names the language files holding that reading, filled from phase 3's reading index (3a) and left empty for a reading no file holds (a caller's own vocabulary, a shape, position alone, and the Korean surname list, which stays outside the per-word files); it too takes `ADDED_IN` and is not compared, `Reading` having shipped in 2.7. The field takes `ADDED_IN` metadata, so a 2.x pickle loads with `None` (the pickle guard as widened in 2.7), and is left out of comparison (`compare=False`), so `==` between two `ParsedName`s means what it means in 2.x. `ReadingKind` gains `PATRONYMIC`, its docstring already allowing new kinds. +- **`Token.reading`**: a `Token` field holding the `Reading` the parse took, the same type `Ambiguity.alternative` holds the reading not taken in (decisions/A1), `None` for a token `replace()` or `revise()` builds. Its `kind` says what the word was read as, its `source` why, and a `languages` field, added to `Reading`, names the language files holding that reading, filled from phase 3's reading index (3a) and left empty for a reading no file holds (a word a caller's own `Lexicon` adds that no file holds in that category, a shape, position alone, and the Korean surname list, which stays outside the per-word files); it too takes `ADDED_IN` and is not compared, `Reading` having shipped in 2.7. The field takes `ADDED_IN` metadata, so a 2.x pickle loads with `None` (the pickle guard as widened in 2.7), and is left out of comparison (`compare=False`), so `==` between two `ParsedName`s means what it means in 2.x. `ReadingKind` gains `PATRONYMIC`, its docstring already allowing new kinds. - **Generation and credentials** (#326): `generation`, the suffix tokens whose reading is a generation or an agnomen (`Jr.`, `III`, `Filho`), and `credentials`, the rest of the suffix (`PhD`, `Esq.`, the honorifics after a name), so the two partition `suffix` and `suffix` is unchanged. CLDR has only these two buckets; the post-nominals that are neither (`Esq.`, the CJK and Hebrew honorifics, `Rinpoche`) fall in `credentials`, CLDR's catch-all, and a caller who wants them apart reads `Token.reading.kind`, which says `post-nominal`. Measured 2026-10-28 on `c8357e74`, over the committed corpus's 6,533 distinct entries (`snapshot.corpus_entries()`) each parsed under its declared order: of the 1,054 with a suffix, 33 hold a generation (or agnomen) and a listed credential together, 77 a generation beside any other suffix word, and 64 only listed post-nominals of other kinds. Of the 615 CLDR renderings with a generation or credentials field, 75 parse with exactly CLDR's suffix words, and in all 75 a split by the vocabulary's reading kind gives CLDR's `generation` and `credentials` both; the other 540 differ in which words the suffix holds, a parse question the split cannot fix. Recompute: classify each suffix word (a token tagged `joined` merged into the one before it) by the kinds of its suffix readings in nameparser/vocab/, matched lowercased with its periods and spaces removed, skipping `default = false` readings, a generation or agnomen reading winning over a credential and a credential over the other kinds, an unlisted roman numeral `i` to `x` a generation and any other unlisted word unlisted; over the corpus entries above, and over `cldr_fields.jsonl` for CLDR. - **A patronymic** (decision 8): `patronymic`, the tokens whose reading is `PATRONYMIC`, in whichever role the parse put them, with `family` and `middle` unchanged. What fills it is a word or a rule that says patronymic, never a guess from position: the vocabulary's `patronymic` particles (`bin`, `binti`, `ibn`, `bint`, Hebrew `bar` and `bat` and the native-script forms) with the name after them, in the default, so `Mahathir bin Mohamad` reads family `bin Mohamad` as today and patronymic `bin Mohamad` beside it; the East Slavic and Turkic rules where RU or TR_AZ turns them on, so `Ivan Ivanovich Ivanov` under RU keeps middle `Ivanovich` and adds patronymic `Ivanovich`; and an Icelandic ending rule an `is` pack would turn on, the default unable to tell `Karlsdóttir` from the Swedish surname `Johansson`. The name `patronymic` is kept for a matronymic too, which is written the same way and cannot be told from one. **`family` is not emptied** by recognizing a patronymic, revising what this section said before: the people filling in the fielded corpora put the patronymic in the family box (below), as CLDR's samples put an Icelandic patronymic in `surname` or `surname-core` (CLDR's `is` file's 3 Icelandic samples, in all 9 of their renderings; its one foreign sample puts a `-dóttir` word in `surname2`) and a Russian one in `given2` (its 2 `ru` samples, in all 6 renderings), our `middle`, while Wikidata records an Icelandic name as a patronymic with no family name; a `family` that keeps it and a `patronymic` that names it satisfy both. A caller writing "Mr. {family}" reads `patronymic` to see that the family is one. - **A second surname**: `surname2`, the family's second name unit where a rule has read the family as two (rules/O6, which a caller or an `es` pack turns on), and empty otherwise, `family` keeping both units as O6 makes it today. O6 already knows which unit it took, so it marks where the second one starts with a new stable tag; a family comma alone marks nothing, since `Lloyd Webber, Andrew` is not two surnames, and a hyphenated family is one token and stays whole. Whether the first unit gets a view of its own, and its name (`surname` sits close to the `surnames` that exists), is a value question below. `surnames` keeps its meaning, `middle` and `family` joined, the reading of a double surname the default splits. @@ -385,7 +385,7 @@ Each sub-phase is one PR, measured on its own base as phase 2's and phase 3's ar | Sub-phase | What | Needs | |---|---|---| -| 4a | `Token.reading` and `ReadingKind.PATRONYMIC`, kind and source read at assemble from each token's final role and the tags the deciding stages left; `Reading.languages` once 3a's index exists. Landed (decisions/vocabulary-model's two 2026-10-29 #754 entries) | — (neutral); 3a for `languages` | +| 4a | `Token.reading` and `ReadingKind.PATRONYMIC`, kind and source read at assemble from each token's final role and the tags the deciding stages left; `Reading.languages` once 3a's index exists. Landed (decisions/vocabulary-model's 2026-10-29 #754, sub-phase 4a and 4a's follow-up entries) | — (neutral); 3a for `languages` | | 4b | `generation` and `credentials` (#326); a listed suffix word's kind read from 3a's index, which 4a could not split (`Jr` and `Esq.` both read `post-nominal` there) | 4a (neutral); 3a for the kind | | 4c | `patronymic`: the vocabulary's patronymic particles, and the RU and TR_AZ rules writing the reading | 4a (neutral) | | 4d | `surname2`: O6's stable tag | — (neutral) | @@ -400,7 +400,7 @@ Each sub-phase is one PR, measured on its own base as phase 2's and phase 3's ar A shape question, decided 2026-10-28: -- **Whether `Lexicon` keeps a public field for each compiled set**, or answers word, kind and property queries from phase 3's reading index and keeps its sets internal. `dataclasses.fields(Lexicon)` gives 22 public fields on 2026-10-28, 11 of them added since 2.0.0's 11 (`surnames`, `honorific_tails`, `conjunctions_ambiguous`, `after_given_titles`, `suffix_words_ambiguous`, `agnomens`, `given_name_abbreviations`, `titles_ambiguous`, `title_modifiers`, `title_styles`, `title_phrases`), each a new parse behavior's set, and a rule keyed on a title kind or on one language's reading of a kind (phase 3's design, "a property a language's own reading should take only under its pack") would add more. Keeping the fields: each is frozen API, shown in the bounded repr (`Lexicon(default + titles: +1)`), and pickled, where `Lexicon`'s own `__setstate__`, unlike the `ADDED_IN` guard the other types share, refuses any pickle whose fields differ, so every new field breaks every older `Lexicon` pickle, and with it every pickled `Parser` and locale pack, until that guard learns the marker. Turning them internal: a break that has to land in 3.0, where a query added beside the fields is an addition that can come later; a caller who configures vocabulary by the field today (`Lexicon.add(titles=...)`, `dataclasses.replace`) needs another way in, for words that carry no kind and no language; and phase 3's reading index, which a `Lexicon` answering queries would read, is loaded on its first lookup (decided lazy on 2026-10-29 rather than loaded with `Lexicon.default()`; about 0.6 ms with its bytecode cached, by phase 3's timing above). Since 4a's follow-up filled `Reading.languages` (2026-10-29), the first parse that reads a listed word makes one, so answering queries from it adds no load beyond that first lookup. The parser need not pay either way: a frozenset test per word costs no frame and an internal cache keeps that path, while a query method called per word would add one per token, the cost the frame budget counts as scaling with the input. **Derek decided on 2026-10-28 that a caller configures vocabulary by reading and the compiled sets become internal** (decisions/vocabulary-model's 2026-10-28 #754 entry on the `Lexicon` surface). A caller states what a word may be in the language files' terms, a `given` reading on `guru`, no credential reading on `dc`, and `Lexicon` derives the sets and gates the build derives, so making `Guru` a possible given name needs no knowledge of rules/H9's sets, where today it is `remove(titles={'guru'}).add(titles_ambiguous={'guru'})`, and `add(given_name_titles=...)` or `add(suffix_acronyms_ambiguous=...)`, the plausible wrong guesses, do something else. A reading is also what a caller can know about a word and what a pack contribution states. Removing a reading from a caller's own lexicon may settle a clash; decision 15 binds the default and the packs, not a caller's choice about their own data. It breaks the frozen `Lexicon` (decisions/3-0-reevaluations); the method names and whether a caller's own language file can join the pool later (phase 3 defers it) are open, and the 2.x warning the freeze asks for was waived on 2026-10-29, no 2.x feature release being left to carry it (decisions/3-0-reevaluations, THE LEXICON WARNING). +- **Whether `Lexicon` keeps a public field for each compiled set**, or answers word, kind and property queries from phase 3's reading index and keeps its sets internal. `dataclasses.fields(Lexicon)` gives 22 public fields on 2026-10-28, 11 of them added since 2.0.0's 11 (`surnames`, `honorific_tails`, `conjunctions_ambiguous`, `after_given_titles`, `suffix_words_ambiguous`, `agnomens`, `given_name_abbreviations`, `titles_ambiguous`, `title_modifiers`, `title_styles`, `title_phrases`), each a new parse behavior's set, and a rule keyed on a title kind or on one language's reading of a kind (phase 3's design, "a property a language's own reading should take only under its pack") would add more. Keeping the fields: each is frozen API, shown in the bounded repr (`Lexicon(default + titles: +1)`), and pickled, where `Lexicon`'s own `__setstate__`, unlike the `ADDED_IN` guard the other types share, refuses any pickle whose fields differ, so every new field breaks every older `Lexicon` pickle, and with it every pickled `Parser` and locale pack, until that guard learns the marker. Turning them internal: a break that has to land in 3.0, where a query added beside the fields is an addition that can come later; a caller who configures vocabulary by the field today (`Lexicon.add(titles=...)`, `dataclasses.replace`) needs another way in, for words that carry no kind and no language; and phase 3's reading index, which a `Lexicon` answering queries would read, is loaded on its first lookup (decided lazy on 2026-10-29 rather than loaded with `Lexicon.default()`; about 0.6 ms with its bytecode cached, by phase 3's timing above). Since 4a's follow-up filled `Reading.languages` (2026-10-29), the first parse that reads a word from its `Lexicon`'s vocabulary makes one, so answering queries from it adds no load beyond that first lookup. The parser need not pay either way: a frozenset test per word costs no frame and an internal cache keeps that path, while a query method called per word would add one per token, the cost the frame budget counts as scaling with the input. **Derek decided on 2026-10-28 that a caller configures vocabulary by reading and the compiled sets become internal** (decisions/vocabulary-model's 2026-10-28 #754 entry on the `Lexicon` surface). A caller states what a word may be in the language files' terms, a `given` reading on `guru`, no credential reading on `dc`, and `Lexicon` derives the sets and gates the build derives, so making `Guru` a possible given name needs no knowledge of rules/H9's sets, where today it is `remove(titles={'guru'}).add(titles_ambiguous={'guru'})`, and `add(given_name_titles=...)` or `add(suffix_acronyms_ambiguous=...)`, the plausible wrong guesses, do something else. A reading is also what a caller can know about a word and what a pack contribution states. Removing a reading from a caller's own lexicon may settle a clash; decision 15 binds the default and the packs, not a caller's choice about their own data. It breaks the frozen `Lexicon` (decisions/3-0-reevaluations); the method names and whether a caller's own language file can join the pool later (phase 3 defers it) are open, and the 2.x warning the freeze asks for was waived on 2026-10-29, no 2.x feature release being left to carry it (decisions/3-0-reevaluations, THE LEXICON WARNING). Value questions inside the shapes above: diff --git a/nameparser/_pipeline/AGENTS.md b/nameparser/_pipeline/AGENTS.md index 0d4bec9d5..ff9f8fc4a 100644 --- a/nameparser/_pipeline/AGENTS.md +++ b/nameparser/_pipeline/AGENTS.md @@ -44,7 +44,7 @@ And work a report does must not walk the name: the given-name title test, asked ### The reading taken is read at assemble -`Token.reading` (3.0, #754's 4a) is not written by the deciding stages: assemble reads it from each token's final role and the tags the deciding stages left (classify's, the `joined` mark group and post_rules write, post_rules' unjoined marks), since a reading written where a word first takes its role would go stale at every later move (P1's fold, P6's attachment, the policy folds). The suffix and name halves ask `_vocab.suffix_kind` and `_vocab.name_source`, the predicates `suffix_reading` and `name_reading` use for the reading not taken, so a change to which reading a TAG offers changes both through one function there; the bound given name's source and the title, connective, particle and initial readings are assemble's own tests, kept out of the shared predicates because an arm added there moves every alternative they name (4a's first draft moved `Smith, John Abu`'s), the initial read off the `initial` tag where `name_reading` tests the shape. A new tag that says what a word was read as, or a decision whose outcome no tag and no role records, needs assemble taught too, with a row in `tests/v2/pipeline/test_assemble.py::_EXPECTED_READINGS`, which holds one row per branch assemble names a reading by; `test_every_parsed_token_has_a_reading_of_its_own_field` checks every case row has one, in its own field, and keeps it through `capitalized()`. decisions/vocabulary-model's 2026-10-29 #754 entry has which role reads as which reading. A reading's `languages` is assemble's too, looked up in the reading index by the word and the kind taken, never by a deciding stage: a new kind needs a row in `_assemble._CATEGORIES` naming the vocabulary categories that hold it, or it names no file, and a new way of writing a listed word (a split, a join) needs a row in `_EXPECTED_LANGUAGES`; the entry after that one has how a word is looked up. +`Token.reading` (3.0, #754's 4a) is not written by the deciding stages: assemble reads it from each token's final role and the tags the deciding stages left (classify's, the `joined` mark group and post_rules write, post_rules' unjoined marks), since a reading written where a word first takes its role would go stale at every later move (P1's fold, P6's attachment, the policy folds). The suffix and name halves ask `_vocab.suffix_kind` and `_vocab.name_source`, the predicates `suffix_reading` and `name_reading` use for the reading not taken, so a change to which reading a TAG offers changes both through one function there; the bound given name's source and the title, connective, particle and initial readings are assemble's own tests, kept out of the shared predicates because an arm added there moves every alternative they name (4a's first draft moved `Smith, John Abu`'s), the initial read off the `initial` tag where `name_reading` tests the shape. A new tag that says what a word was read as, or a decision whose outcome no tag and no role records, needs assemble taught too, with a row in `tests/v2/pipeline/test_assemble.py::_EXPECTED_READINGS`, which holds one row per branch assemble names a reading by; `test_every_parsed_token_has_a_reading_of_its_own_field` checks every case row has one, in its own field, and keeps it through `capitalized()`. decisions/vocabulary-model's 2026-10-29 #754, sub-phase 4a entry has which role reads as which reading. A reading's `languages` is assemble's too, looked up in the reading index by the word and the kind taken, never by a deciding stage: a new kind needs a row in `_assemble._CATEGORIES` naming the vocabulary categories that hold it, or it names no file, and a new way of writing a listed word (a split, a join) needs a row in `_EXPECTED_LANGUAGES`; its 2026-10-29 #754, 4a's follow-up entry has how a word is looked up. ### Testing an emitter diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py index 375276a1c..868fee03d 100644 --- a/nameparser/_pipeline/_assemble.py +++ b/nameparser/_pipeline/_assemble.py @@ -40,7 +40,7 @@ AGNOMEN_TAG, AMBIGUOUS_ACRONYM_TAG, AMBIGUOUS_WORD_TAG, FIELD_READING, NAME_ROLES, SHAPE_ACRONYM_TAG, SPLIT_CREDENTIAL_TAG, SPLIT_MEMBER_TAG, TITLE_AMBIGUOUS_TAG, Beside, Deferred, ParseState, PendingAmbiguity, - Structure, + Structure, WorkToken, ) from nameparser._pipeline._vocab import ( _PERIOD_ABBREV, name_source, suffix_kind, @@ -117,10 +117,26 @@ def _by_parts(n: str, cats: frozenset[str], keep: str) -> tuple[str, ...]: parts = "".join(" " if c not in keep and unicodedata.category(c)[0] in "PSZ" else c for c in n).split() - return tuple(dict.fromkeys( + return tuple(sorted({ tag for part in parts for tag in languages_of(part.replace(keep, "") if keep else part, - cats))) + cats)})) + + +def _split_partner(tokens: tuple[WorkToken, ...], dropped: set[int], + i: int) -> str: + """The text of the second word of the credential split across a + space that token `i` opens: the next word past a nickname written + between them ('Ph. "Bob" D.'), or nothing where that word is not + the joined suffix word that completes it. Asked only of a split + credential's first word, so other names pay nothing.""" + for j in range(i + 1, len(tokens)): + if j in dropped or tokens[j].role is Role.NICKNAME: + continue + nxt = tokens[j] + return (nxt.text if nxt.role is Role.SUFFIX and "joined" in nxt.tags + else "") + return "" _reading = functools.lru_cache(maxsize=1024)(Reading) @@ -168,10 +184,12 @@ def _alternative(pending: PendingAmbiguity, materialized: tuple[Token, ...], role = Role.FAMILY # the files holding the reading for any of its words, as a token's # own reading names them - langs = (tuple(dict.fromkeys( - tag for t in materialized for tag in _languages(t.text, alt.kind))) - if alt.source in _LISTED else ()) - return Reading(alt.kind, role, alt.source, langs) + found: set[str] = set() + if alt.source in _LISTED: + # a loop, not a generator, which would resume a frame per word + for t in materialized: + found.update(_languages(t.text, alt.kind)) + return Reading(alt.kind, role, alt.source, tuple(sorted(found))) def _opens_its_segment(state: ParseState, i: int) -> bool: @@ -263,15 +281,18 @@ def assemble(state: ParseState) -> ParsedName: # a credential split across a space is listed whole # ('Ph.' 'D.' as 'phd'): its first word reads the files # of the two words, which the second then takes - written = (t.text + tokens[i + 1].text - if prev_split and i + 1 < last else t.text) + written = (t.text + _split_partner(tokens, dropped, i) + if prev_split else t.text) reading = (_reading(kind, role, source, langs) if source in _LISTED and (langs := _languages(written, kind)) else _READINGS[kind, role, source]) before = reading else: - before = None + if role is not Role.NICKNAME: + # a nickname written inside a split credential ('Ph. + # "Bob" D.') leaves the run to its second word + before = None if role is Role.TITLE: if "conjunction" in tags: kind, source = (ReadingKind.CONJUNCTION, diff --git a/nameparser/_types.py b/nameparser/_types.py index dff34aecd..94d433b2e 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -950,11 +950,12 @@ class Reading: #: shape, or position alone. source: ReadingSource #: The BCP 47 tags of the shipped vocabulary files that hold this - #: reading of the word, on by default (``("es", "fr", "it", "pt")`` - #: for the particle ``de``, ``("mul",)`` for the title ``Dr.``), in - #: tag order. Empty for a reading no file holds: a shape, position - #: alone, a caller's own vocabulary, or a shipped list kept outside - #: the per-word files (the Korean surnames that divide ``김민준``). + #: reading of the word, on by default (``("es", "fr", "it", "nl", + #: "pt")`` for the particle ``de``, ``("mul",)`` for the title + #: ``Dr.``), sorted. Empty for a reading no file holds: a shape, + #: position alone, a word a caller's own ``Lexicon`` adds that no + #: file holds in that category, or a shipped list kept outside the + #: per-word files (the Korean surnames that divide ``김민준``). #: A suffix reading is matched by the word's suffix entry whatever #: its finer kind, a word having one. An ``Ambiguity.alternative`` #: names the files holding it for any word the report names. Not diff --git a/tests/v2/pipeline/test_assemble.py b/tests/v2/pipeline/test_assemble.py index 15f6e86fd..3a02264d0 100644 --- a/tests/v2/pipeline/test_assemble.py +++ b/tests/v2/pipeline/test_assemble.py @@ -360,6 +360,9 @@ def test_every_parsed_token_has_a_reading_of_its_own_field(case: Case) -> None: repaired = parser.capitalized(pn, force=True) assert ([t.reading for t in repaired.tokens] == [t.reading for t in pn.tokens]) + # languages is not compared, so it is checked on its own + assert ([t.reading and t.reading.languages for t in repaired.tokens] + == [t.reading and t.reading.languages for t in pn.tokens]) def test_a_caller_surname_reads_from_the_vocabulary_only_where_split_off( @@ -446,11 +449,17 @@ def test_a_suffix_word_reads_as_its_vocabulary_kind( ("John Smith Ph. D.", "D.", ("en",)), ("Smith, Ph. D. III", "D.", ("en",)), ("John Smith Pharm D", "Pharm", ("en",)), + # a nickname written between a split credential's words is passed + # over, and the run reaches its second word + ('John Smith Ph. "Bob" D.', "Ph.", ("en",)), + ('John Smith Ph. "Bob" D.', "D.", ("en",)), # a word joining listed words, by its parts; never divided at a # combining mark ("John Smith, M.D./Ph.D.", "M.D./Ph.D.", ("en",)), ("Lt.Gov. John Smith", "Lt.Gov.", ("en",)), ("प्रोफे.डॉ. अदा शर्मा", "प्रोफे.डॉ.", ("hi", "mr")), + # a union over parts is sorted, as a word's own files are + ("John Smith, MS-V", "MS-V", ("en", "mul")), # no file: position, shape, and a shipped list outside the per-word # files (the Korean surnames) ("Dr. Juan de la Vega III", "Juan", ()), @@ -491,3 +500,7 @@ def test_an_alternative_names_the_files_holding_it() -> None: (amb,) = parse("van Gogh").ambiguities assert amb.alternative is not None assert amb.alternative.languages == ("nl",) + # over the report's words, sorted ('van' nl, 'DO' en) + (amb,) = parse("Doe, John van DO").ambiguities + assert amb.alternative is not None + assert amb.alternative.languages == ("en", "nl") From dce56273a40239fe3115eeabc88ab98eb352df50 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 29 Oct 2026 14:26:40 -0700 Subject: [PATCH 4/7] Record the nickname split's reading and correct 3a's load forecast (#754) Co-Authored-By: Claude Opus 5.5 --- docs/design/decisions/vocabulary-model.md | 4 ++-- docs/design/vocabulary-model-rfc.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index 3aff947c3..4bb300ec2 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -280,7 +280,7 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) - **A name reading written `default = false` derives no marker in the default** (item 3, the half before a pack turns it on): the word's other readings compile as they would without it. So it counts for none of the checks guarding the default's compile: an agnomen on by default needs a name reading on by default (check 11, else it reads ungated), and a bound given name that is also a particle, which needs a `given` reading on by default (else Lexicon raises at import), is refused by check 11's particle half below before check 9 reads it. A name reading off beside a suffix word kind no gate reads is still refused, gating nothing when on either. - **Left open: a particle whose word's only `given` readings are off is refused.** Check 11 asks such a particle to state `may_be_given = false` exactly where its word has no `given` reading. The default compiles it never-given, a pack turning the given reading on compiles it possibly given, and which of the two the stated property describes is a choice no 3.0 pack needs yet (`nl`'s prepositions wait with strict). The build refuses the shape, naming both ways out, rather than pick a side; the first pack that needs it decides, with its measurement. - **The index is written grouped by word, and an `IndexReading` is built only for a word looked up.** On a flat draft of this change, one row per line, grouping the rows at run time took about 0.56 ms; building every row's object takes about 1.4 ms (`index()`, which only a compile over the whole pool calls); each against the import's own cost, so the build writes one line per word and the lookup table is a single `dict()` call. A change to a word's readings is still a one-line diff. - - **The index loads on the first `readings_of` call, not inside `Lexicon.default()` (decided 2026-10-29, Derek).** SUPERSEDES: item 5's "loaded for every caller with `Lexicon.default()`". Proposed by 3a and chosen over loading it with the default lexicon. What it buys depends on the state of `Reading.languages`. Today, and until the follow-up that fills it after 3a and 4a (#942) both merge, no parse reads the index, so no caller pays the import, parsing or not, where loading it with the default lexicon would cost every parsing caller the import for nothing. Once the follow-up fills `Reading.languages`, the first parse will load it, so every parsing caller will pay it once, as item 5 measured, and only a caller who imports nameparser without parsing is spared. Wherever the follow-up calls `readings_of`, the import moves with the call. + - **The index loads on the first `readings_of` call, not inside `Lexicon.default()` (decided 2026-10-29, Derek).** SUPERSEDES: item 5's "loaded for every caller with `Lexicon.default()`". Proposed by 3a and chosen over loading it with the default lexicon. What it buys depends on the state of `Reading.languages`. Today, and until the follow-up that fills it after 3a and 4a (#942) both merge, no parse reads the index, so no caller pays the import, parsing or not, where loading it with the default lexicon would cost every parsing caller the import for nothing. Once the follow-up fills `Reading.languages`, the first parse will load it, so every parsing caller will pay it once, as item 5 measured, and only a caller who imports nameparser without parsing is spared. Wherever the follow-up calls `readings_of`, the import moves with the call. (Corrected 2026-10-29: the follow-up calls `languages_of`, which loads the index through the same lookup table, and only a parse that reads a word its own `Lexicon` lists loads it, so `John Smith` does not; "every parsing caller will pay it" was the forecast. The 2026-10-29 #754, 4a's follow-up entry's item 4 has the measurement.) - **Measured** 2026-10-29 on `9210f221` plus this change, Python 3.11.17, medians of fresh processes with bytecode cached as an installed wheel has it: importing the index takes 0.56 to 0.68 ms across three runs on a shared machine, about 3% of `import nameparser`'s 18 to 24 ms in the same runs, above the 0.45 ms item 5 measured on a flat draft of 2,140 rows; the first lookup after it, 0.07 ms; compiling the whole index, 2.1 ms; uncached, the import takes about 13 to 18 ms (13 after `import nameparser` in this session, 17 to 19 in review's runs, a figure no wheel install pays). The index holds 2,157 readings over 1,798 words, 106 of them in more than one file, and 3 readings written `default = false`; it is about 95 KB of source. Recompute: `uv run python tools/vocab/build.py`, then with `PYTHONDONTWRITEBYTECODE` unset, time `import nameparser.config._index` after `import nameparser` in fresh processes (the first run writes the bytecode), and count with `from nameparser.config._index import READINGS`. - Not touched: `Lexicon` and every other pickled type gain no field, so no pickle layout moves, and 3a forces no public name; the configure-by-reading methods stay open. - **2026-10-29 #754, sub-phase 4a — `Token.reading` IS READ AT ASSEMBLE, FROM EACH TOKEN'S FINAL ROLE AND THE TAGS THE DECIDING STAGES LEFT, ITS SUFFIX AND NAME SOURCES BY THE PREDICATES THAT NAME `Ambiguity.alternative`.** It carries out item 2 of the 2026-10-28 #754 entry above, all but `Reading.languages`, which a follow-up adds and fills from 3a's reading index (`_reading_index.readings_of`, the 2026-10-29 #753 entry above), adding the field with its values because an empty one would say that no file holds the reading. @@ -292,7 +292,7 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) - **2026-10-29 #754, 4a's follow-up — `Reading.languages` IS FILLED AT ASSEMBLE FROM 3a's READING INDEX, FOR THE READING A TOKEN TOOK AND FOR A REPORT'S ALTERNATIVE; NO PARSE MOVES.** Completes 4a (the 2026-10-29 #754, sub-phase 4a entry) with the field item 2 of the 2026-10-28 #754 entry names. Until it, no parse read the index (the 2026-10-29 #753, phase 3a entry's load bullet); since it, the first parse that reads a word from the vocabulary of its own `Lexicon` loads it (item 4). SUPERSEDES: decisions/A1's 2026-10-26 #752 (phase 2m) entry's decline of the reading's `languages`, for 3.0; its reason, that the index describes the default and not the `Lexicon` a parse used, is answered by item 1's semantics rather than removed. 1. **What it holds.** The BCP 47 tags of the shipped files holding the reading taken, on by default, each once, sorted (a word joined from parts and an alternative over several words sorting the union), read through `_reading_index.languages_of(text, categories)`, which walks the index rows without building an `IndexReading`. A reading matches a file's rows by category, not by the vocabulary's finer kind: a `title`, `particle` or `conjunction` reading its category's rows, a `name` reading the `name` and `bound_given` rows, and every suffix kind the word's suffix rows, a word having one suffix kind (the build's check 4), so `Jr`, which reads `post-nominal` until 4b, names `en`, the file of its `generation` reading. Only a `vocabulary` or `declared` reading is looked up; a shape, position, an initial or a patronymic names no file. A reading written `default = false` names none (`St` in a caller's `suffix_words`, held off by `cs.toml`), and neither does a word a caller's own `Lexicon` adds that no shipped file holds in that category (`Zzq` as a title), while one a file does hold names it (`Dean` added to `titles` names `en`), nor the Korean surnames that divide unspaced hangul, which `config/surnames.py` keeps outside the per-word files: `김` in `김민준` reads `name`, `vocabulary`, with no files. It names the default's files whatever `Lexicon` the parse used, the semantics decisions/A1's 2026-10-26 decline weighed against it: a file holds the reading or not, and which lexicon read it is the caller's own knowledge. Under a choice of languages (3b) a "live" reading becomes one the choice makes, and `languages_of` filtering on `on_by_default` is the place 3b changes. - 2. **How a word is looked up.** By the vocabulary's own form of the word (`_normalize`); a suffix also with every period removed, as a credential is matched (`M.A` in `John Smith (M.A)`); a credential split across a space whole, its first word joined to the second (`Ph.` `D.` as `phd`), the second taking the first's reading; and a word joining listed words with punctuation by its parts, in order, a suffix's parts divided by any mark but the period first (`M.D./Ph.D.`, `MD;`), then any word's by every mark (`Lt.Gov.`), never at a combining mark, which `\W` would count (`प्रोफे.डॉ.` names `hi` and `mr` through `डॉ`). An alternative names the files for any word its report names. MEASURED 2026-10-29 on `bd02d68f` plus this change over the snapshot's 6,360 distinct names, the empty one among them (20,804 tokens): 3,935 readings are `vocabulary` or `declared`, 3,778 of them name a file, and the 157 that name none are all Korean surnames split off by the list; by the normalized form and the suffix's period-free form alone, 265 would name none: the 157, 42 words joining listed words, and 66 words of credentials split across a space, 40 by `split:credential` (`Ph.` `D.`) and 26 by `split:member` (`Pharm D`). Recompute: parse every `name` of `tools/differential/snapshot.jsonl` and count the tokens whose `reading.source` is `vocabulary` or `declared` by whether `reading.languages` is empty; for the second figure, look each up with `languages_of` by its normalized and period-free forms only, and sort the misses by the split tags on the word and the word before it. + 2. **How a word is looked up.** By the vocabulary's own form of the word (`_normalize`); a suffix also with every period removed, as a credential is matched (`M.A` in `John Smith (M.A)`); a credential split across a space whole, its first word joined to the second (`Ph.` `D.` as `phd`), the second taking the first's reading, past a nickname written between them too (`John Smith Ph. "Bob" D.`), where the second word read `credential`, `position` under 4a alone and now takes the first's `credential`, `vocabulary`: the one `Token.reading` this change moves, no role, tag or alternative moving with it; and a word joining listed words with punctuation by its parts, in order, a suffix's parts divided by any mark but the period first (`M.D./Ph.D.`, `MD;`), then any word's by every mark (`Lt.Gov.`), never at a combining mark, which `\W` would count (`प्रोफे.डॉ.` names `hi` and `mr` through `डॉ`). An alternative names the files for any word its report names. MEASURED 2026-10-29 on `bd02d68f` plus this change over the snapshot's 6,360 distinct names, the empty one among them (20,804 tokens): 3,935 readings are `vocabulary` or `declared`, 3,778 of them name a file, and the 157 that name none are all Korean surnames split off by the list; by the normalized form and the suffix's period-free form alone, 265 would name none: the 157, 42 words joining listed words, and 66 words of credentials split across a space, 40 by `split:credential` (`Ph.` `D.`) and 26 by `split:member` (`Pharm D`). Recompute: parse every `name` of `tools/differential/snapshot.jsonl` and count the tokens whose `reading.source` is `vocabulary` or `declared` by whether `reading.languages` is empty; for the second figure, look each up with `languages_of` by its normalized and period-free forms only, and sort the misses by the split tags on the word and the word before it. 3. **The field.** `Reading.languages: tuple[str, ...] = ()`, `compare=False`, so two readings compare as in 2.x and every recorded alternative is untouched, and `metadata={ADDED_IN: "3.0.0"}`, `Reading` having shipped in 2.7: a `Reading` pickled by 2.7 or 2.8 loads with `()` (`tests/v2/data/parsedname_v28_alternative.pickle`, made by the released 2.8.0 wheel), and a renamed field or a shipped one missing beside it is still refused (`_REFUSED_SKEWS`). The constructor takes any iterable of str and refuses a bare str and a non-str tag. The repr names the files where there are any (`Reading('particle' as 'family', 'vocabulary', in es fr it nl pt)`). The snapshot does not record readings, so nothing re-records; it is byte-identical. 4. **Cost**, measured 2026-10-29, Python 3.11.17. Two memos keep the lookups off the frame count: `_languages`, an `lru_cache(maxsize=4096)` keyed on the word as written and the kind, and an `lru_cache(maxsize=1024)` over `Reading` for the readings that name files, each a pure function of the shipped files, bounded because the caller's text keys them, and C-level on a hit, so a word read before costs no frame. `tools/perf/call_count.py`'s reference row moved from 311.00 on `bd02d68f` to 311.40, all of it the first reference parse of the 50 (331 frames against 311: the index import and the four first lookups); after it each parse counts 311.0 on both trees. `_CALL_BASELINE` (307) is not moved and the serial benchmark passes. Which parse loads the index is decided by the parse's own `Lexicon`, not the files: any `vocabulary` or `declared` reading looks its word up, so `김민준` and a caller's own title (`Zzq John Smith`) load it though no file holds either, while `John Smith` does not. In fresh processes with bytecode cached, as an installed wheel has it, the first parse to read a listed word (`Dr. Juan de la Vega III`, after a `John Smith`) takes 1.20 to 1.22 ms against 0.22 ms on `bd02d68f`, the medians of 21 processes in each of three runs: about 1.0 ms once per process, the index import 3a measured at 0.56 to 0.68 ms plus the first lookups and readings. The steady parse, over 4,000 names half of them listed-word names, takes 78.6 to 79.0 µs against 78.3 to 79.4 µs on `bd02d68f`, within the runs' noise. Recompute: time the two parses in fresh processes and a `timeit` over the 4,000 names in each tree, with `PYTHONDONTWRITEBYTECODE` unset and the index's bytecode written. 5. **Not done here: a listed suffix word's kind.** The index lookup this change makes does not settle 4b's `Jr` and `Sr` (`tests/v2/pipeline/test_assemble.py`'s strict expected failures stand): the kind a token takes is named by `_vocab.suffix_kind`, which also names `Ambiguity.alternative`, so reading the kind from the index at assemble alone would split the two, and reading it inside `suffix_kind` moves the alternatives and puts the index on the decision path. 4b decides it. diff --git a/docs/design/vocabulary-model-rfc.md b/docs/design/vocabulary-model-rfc.md index 432370140..d3b4a194c 100644 --- a/docs/design/vocabulary-model-rfc.md +++ b/docs/design/vocabulary-model-rfc.md @@ -219,7 +219,7 @@ The 2 broken are pairs whose romanization opens with a listed title, which reads ### Where the provenance lives at runtime -The compiled module holds sets, and a set has forgotten which file each word came from. Phase 3 compiles a second module beside it (decided 2026-10-28): a **reading index**, one row per reading (its text, its file's tag, its kind and its behavioral properties), loaded once per process on its first lookup (`_reading_index.readings_of`) (decided 2026-10-29, replacing the 2026-10-28 decision to load it with `Lexicon.default()`). Phase 4's `Reading.languages` is filled from it at assemble (4a's follow-up, decisions/vocabulary-model's 2026-10-29 #754 entry on it), so the first parse in a process that reads a word from its `Lexicon`'s vocabulary loads it (a listed or declared word, a Korean surname dividing hangul or a caller's own word, whether or not a file holds it): every caller who parses such a name pays it once, about 1 ms on that parse, and a caller who only imports nameparser, or parses only words its `Lexicon` does not list, does not. Filling `reading.languages` on first access to that field instead was weighed and set aside: a `Reading` holds its kind, role and source but not its word, so a lazy field would have to carry the word as a hidden key on a frozen, slotted, pickled type, to save a cost the measurement below puts at about 2% of the import. The compiler moves from tools/vocab/build.py into the package so that the build and a pack share one implementation, and a test holds the invariant that compiling the index with no choice reproduces `nameparser/config/_compiled.py` exactly. The same index is what item 13 of decisions/vocabulary-model's 2026-10-23 #752 entry says `Token.reading` needs in 3.0; 3a built it (decisions/vocabulary-model, 2026-10-29), its import measured there at 0.56 to 0.68 ms. +The compiled module holds sets, and a set has forgotten which file each word came from. Phase 3 compiles a second module beside it (decided 2026-10-28): a **reading index**, one row per reading (its text, its file's tag, its kind and its behavioral properties), loaded once per process on its first lookup (`_reading_index.readings_of`) (decided 2026-10-29, replacing the 2026-10-28 decision to load it with `Lexicon.default()`). Phase 4's `Reading.languages` is filled from it at assemble (4a's follow-up, decisions/vocabulary-model's 2026-10-29 #754, 4a's follow-up entry), so the first parse in a process that reads a word from its `Lexicon`'s vocabulary loads it (a listed or declared word, a Korean surname dividing hangul or a caller's own word, whether or not a file holds it): every caller who parses such a name pays it once, about 1 ms on that parse, and a caller who only imports nameparser, or parses only words its `Lexicon` does not list, does not. Filling `reading.languages` on first access to that field instead was weighed and set aside: a `Reading` holds its kind, role and source but not its word, so a lazy field would have to carry the word as a hidden key on a frozen, slotted, pickled type, to save a cost the measurement below puts at about 2% of the import. The compiler moves from tools/vocab/build.py into the package so that the build and a pack share one implementation, and a test holds the invariant that compiling the index with no choice reproduces `nameparser/config/_compiled.py` exactly. The same index is what item 13 of decisions/vocabulary-model's 2026-10-23 #752 entry says `Token.reading` needs in 3.0; 3a built it (decisions/vocabulary-model, 2026-10-29), its import measured there at 0.56 to 0.68 ms. Measured 2026-10-27 on `cfe8bf92`: the pool holds 2,140 readings over 1,797 words, 106 of them in more than one file. An index of their behavioral properties is about 93 KB of generated source. Imported with its bytecode cached, as an installed wheel has it, it takes about 0.45 ms (median of nine fresh processes, 2026-10-28 on `c8357e74`, Python 3.11.17), against about 20 ms for `import nameparser` cached the same way; with no cache it takes about 11 ms, the figure this section gave on 2026-10-27, measured in a shell that sets `PYTHONDONTWRITEBYTECODE`. That load is paid once per process, at the first lookup. Compiling the pool takes about 9 ms, paid once per choice and only by a caller who makes one. Reading the TOML instead costs about 39 ms. Recompute: the build's `read_dir`, `load` and `compile_pool`, timed; for the import, write the index as a module, `py_compile` it, and time its import in fresh processes beside a cached `import nameparser`. From 2aab54688e3f4e7b7f0b130890a2f24dd62b9bf1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 29 Oct 2026 14:45:08 -0700 Subject: [PATCH 5/7] Name a name reading's files by its field; read a caller's whole word whole (#754) Answers the orchestrator's review of #944. A name reading now takes the rows listing the word as a name of the field it lands in, so 'Ma' as a family name names en and not fil, which lists it only as given. A word joining listed words is read by its parts only where the parse's own Lexicon does not list it whole, so a caller's 'zzq-dr' title names none of dr's files, and a word listed whole without its periods ('Lt.Gen.' as 'ltgen') is found whole. Reading.languages refuses a mapping and a non-iterable with a message naming the fix. The docs give the uncached first-load cost beside the cached one and name languages_of as the lookup a parse makes. Co-Authored-By: Claude Opus 5.5 --- docs/design/decisions/vocabulary-model.md | 10 +- docs/design/vocabulary-model-rfc.md | 2 +- docs/release_log.rst | 2 +- docs/usage.rst | 3 +- nameparser/_pipeline/AGENTS.md | 2 +- nameparser/_pipeline/_assemble.py | 153 +++++++++++++++------- nameparser/_reading_index.py | 15 +-- nameparser/_types.py | 19 ++- tests/v2/pipeline/test_assemble.py | 31 ++++- tests/v2/test_reprs.py | 2 +- tests/v2/test_types.py | 20 ++- 11 files changed, 180 insertions(+), 79 deletions(-) diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index 4bb300ec2..5b9ceb561 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -20,7 +20,7 @@ The vocabulary was a set of base sets plus marker subsets (`particles` and `part - Name readings are hand-declared, and only on a word some language file claims as vocabulary or refuses, so the vocabulary carries no surname list (the segmentation lists, Korean and the ZH pack's, are outside the per-word model) — 2026-10-15/16; 2026-10-19. - Bearer counts are structured provenance (source and count), never used to compute a reading — 2026-10-15/16. - Phase 2 is to put vocabulary, declared name and shape readings in one pool, derive `may_be_given` and `may_be_name` from declared name readings (a count-free one carrying a `note`, `ld` left ungated, `may_be_initial` kept), and decide every clash by one gate of four steps, floor, form, structure and count, asked at the sites that decide today; the reading taken is to surface as `Token.reading` in 3.0 — 2026-10-23 #752. The reading not taken surfaces as `Ambiguity.alternative` since 2.7 (decisions/A1) — 2026-10-26 #752, phase 2m. -- Every reading the parse took or a report names from the vocabulary names the shipped files holding it, `Reading.languages`, looked up at assemble in the reading index by the word and the kind; the default's files whatever `Lexicon` the parse used, and none for a shape, position, or a word no file holds in that category — 2026-10-29 #754, 4a's follow-up. +- Every reading the parse took or a report names from the vocabulary names the shipped files holding it, `Reading.languages`, looked up at assemble in the reading index by the word and the kind, a name by the field it lands in; the default's files whatever `Lexicon` the parse used, and none for a shape, position, or a word no file holds in that category — 2026-10-29 #754, 4a's follow-up. - Every numeral rule asks one predicate, `_vocab.reads_as_generation`, the vocabulary's own numerals read inside it; a numeral of two or more letters is the generation wherever the trailing run holds it, while a one-letter one before a credential stays the name word decisions/S10's #721 entry made it — 2026-10-25 #752, phase 2f. - A credential written split across a space is one word, a member of the ambiguous class that the count decides, and the Ph./D. pair starts S8's run from where it stands (decisions/S12) — 2026-10-25 #752, phase 2e. - A listed credential that is also a name, written in capitals in a name of mixed case, is settled as the credential by its writing: it reports nothing and starts the credential's run where no name word stands behind it with no family comma (2026-10-26, Derek), and keeps its other readings, the clash settled at the sites that choose a reading rather than by retagging the word (decisions/S6) — 2026-10-25 #752, phase 2g, #785. @@ -291,8 +291,8 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) 5. **Left open.** A listed suffix word that the compiled sets do not split by kind reads `post-nominal`, so `Jr`, `Sr` and `Esq.` read alike: the TOML gives `jr` a `generation` reading, but `SUFFIX_WORDS` holds both kinds and classify tags them alike, so only a vocabulary lookup by kind can tell them apart: 3a's reading index, a new `Lexicon` field being set aside (the 2026-10-28 `Lexicon` surface entry makes the compiled sets internal, and `Lexicon`'s own pickle guard refuses an added field). It is the predicate's own answer, which `Ambiguity.alternative` has given since 2.7 (the snapshot's one post-nominal alternative is `Rinpoche`'s), and a kind left unset would have meant a parsed token with no reading or a kind for "not yet known", so the reading taken agrees with it until 4b, whose `generation` view needs the split, reads a listed word's kind from 3a's index; `tests/v2/pipeline/test_assemble.py::test_a_suffix_word_reads_as_its_vocabulary_kind` holds `Jr` and `Sr` as strict expected failures until then. `ReadingKind.PATRONYMIC` exists and no reading takes it until 4c, whose rules write it. This amends the 2026-10-28 #754 entry's item 9: 4b needs 3a for that kind. - **2026-10-29 #754, 4a's follow-up — `Reading.languages` IS FILLED AT ASSEMBLE FROM 3a's READING INDEX, FOR THE READING A TOKEN TOOK AND FOR A REPORT'S ALTERNATIVE; NO PARSE MOVES.** Completes 4a (the 2026-10-29 #754, sub-phase 4a entry) with the field item 2 of the 2026-10-28 #754 entry names. Until it, no parse read the index (the 2026-10-29 #753, phase 3a entry's load bullet); since it, the first parse that reads a word from the vocabulary of its own `Lexicon` loads it (item 4). SUPERSEDES: decisions/A1's 2026-10-26 #752 (phase 2m) entry's decline of the reading's `languages`, for 3.0; its reason, that the index describes the default and not the `Lexicon` a parse used, is answered by item 1's semantics rather than removed. - 1. **What it holds.** The BCP 47 tags of the shipped files holding the reading taken, on by default, each once, sorted (a word joined from parts and an alternative over several words sorting the union), read through `_reading_index.languages_of(text, categories)`, which walks the index rows without building an `IndexReading`. A reading matches a file's rows by category, not by the vocabulary's finer kind: a `title`, `particle` or `conjunction` reading its category's rows, a `name` reading the `name` and `bound_given` rows, and every suffix kind the word's suffix rows, a word having one suffix kind (the build's check 4), so `Jr`, which reads `post-nominal` until 4b, names `en`, the file of its `generation` reading. Only a `vocabulary` or `declared` reading is looked up; a shape, position, an initial or a patronymic names no file. A reading written `default = false` names none (`St` in a caller's `suffix_words`, held off by `cs.toml`), and neither does a word a caller's own `Lexicon` adds that no shipped file holds in that category (`Zzq` as a title), while one a file does hold names it (`Dean` added to `titles` names `en`), nor the Korean surnames that divide unspaced hangul, which `config/surnames.py` keeps outside the per-word files: `김` in `김민준` reads `name`, `vocabulary`, with no files. It names the default's files whatever `Lexicon` the parse used, the semantics decisions/A1's 2026-10-26 decline weighed against it: a file holds the reading or not, and which lexicon read it is the caller's own knowledge. Under a choice of languages (3b) a "live" reading becomes one the choice makes, and `languages_of` filtering on `on_by_default` is the place 3b changes. - 2. **How a word is looked up.** By the vocabulary's own form of the word (`_normalize`); a suffix also with every period removed, as a credential is matched (`M.A` in `John Smith (M.A)`); a credential split across a space whole, its first word joined to the second (`Ph.` `D.` as `phd`), the second taking the first's reading, past a nickname written between them too (`John Smith Ph. "Bob" D.`), where the second word read `credential`, `position` under 4a alone and now takes the first's `credential`, `vocabulary`: the one `Token.reading` this change moves, no role, tag or alternative moving with it; and a word joining listed words with punctuation by its parts, in order, a suffix's parts divided by any mark but the period first (`M.D./Ph.D.`, `MD;`), then any word's by every mark (`Lt.Gov.`), never at a combining mark, which `\W` would count (`प्रोफे.डॉ.` names `hi` and `mr` through `डॉ`). An alternative names the files for any word its report names. MEASURED 2026-10-29 on `bd02d68f` plus this change over the snapshot's 6,360 distinct names, the empty one among them (20,804 tokens): 3,935 readings are `vocabulary` or `declared`, 3,778 of them name a file, and the 157 that name none are all Korean surnames split off by the list; by the normalized form and the suffix's period-free form alone, 265 would name none: the 157, 42 words joining listed words, and 66 words of credentials split across a space, 40 by `split:credential` (`Ph.` `D.`) and 26 by `split:member` (`Pharm D`). Recompute: parse every `name` of `tools/differential/snapshot.jsonl` and count the tokens whose `reading.source` is `vocabulary` or `declared` by whether `reading.languages` is empty; for the second figure, look each up with `languages_of` by its normalized and period-free forms only, and sort the misses by the split tags on the word and the word before it. - 3. **The field.** `Reading.languages: tuple[str, ...] = ()`, `compare=False`, so two readings compare as in 2.x and every recorded alternative is untouched, and `metadata={ADDED_IN: "3.0.0"}`, `Reading` having shipped in 2.7: a `Reading` pickled by 2.7 or 2.8 loads with `()` (`tests/v2/data/parsedname_v28_alternative.pickle`, made by the released 2.8.0 wheel), and a renamed field or a shipped one missing beside it is still refused (`_REFUSED_SKEWS`). The constructor takes any iterable of str and refuses a bare str and a non-str tag. The repr names the files where there are any (`Reading('particle' as 'family', 'vocabulary', in es fr it nl pt)`). The snapshot does not record readings, so nothing re-records; it is byte-identical. - 4. **Cost**, measured 2026-10-29, Python 3.11.17. Two memos keep the lookups off the frame count: `_languages`, an `lru_cache(maxsize=4096)` keyed on the word as written and the kind, and an `lru_cache(maxsize=1024)` over `Reading` for the readings that name files, each a pure function of the shipped files, bounded because the caller's text keys them, and C-level on a hit, so a word read before costs no frame. `tools/perf/call_count.py`'s reference row moved from 311.00 on `bd02d68f` to 311.40, all of it the first reference parse of the 50 (331 frames against 311: the index import and the four first lookups); after it each parse counts 311.0 on both trees. `_CALL_BASELINE` (307) is not moved and the serial benchmark passes. Which parse loads the index is decided by the parse's own `Lexicon`, not the files: any `vocabulary` or `declared` reading looks its word up, so `김민준` and a caller's own title (`Zzq John Smith`) load it though no file holds either, while `John Smith` does not. In fresh processes with bytecode cached, as an installed wheel has it, the first parse to read a listed word (`Dr. Juan de la Vega III`, after a `John Smith`) takes 1.20 to 1.22 ms against 0.22 ms on `bd02d68f`, the medians of 21 processes in each of three runs: about 1.0 ms once per process, the index import 3a measured at 0.56 to 0.68 ms plus the first lookups and readings. The steady parse, over 4,000 names half of them listed-word names, takes 78.6 to 79.0 µs against 78.3 to 79.4 µs on `bd02d68f`, within the runs' noise. Recompute: time the two parses in fresh processes and a `timeit` over the 4,000 names in each tree, with `PYTHONDONTWRITEBYTECODE` unset and the index's bytecode written. + 1. **What it holds.** The BCP 47 tags of the shipped files holding the reading taken, on by default, each once, sorted (a word joined from parts and an alternative over several words sorting the union), read through `_reading_index.languages_of(text, kinds)`, which walks the index rows without building an `IndexReading`. A reading matches the rows of every vocabulary kind of its category: a `title`, `particle` or `conjunction` reading its category's rows; a `name` reading the rows listing the word as a name of the field it lands in, `family` for a family or maiden word and `given` for a given or middle one, with the `bound_given` rows in either (a nickname takes any name rows), so `Ma` read as a family name (`Long Ma`) names `en` and not `fil`, which lists it only as a given name, and a declared name read into a field no file lists it in names none (`van` as a family name, which `nl` lists as given); and every suffix kind the word's suffix rows, a word having one suffix kind (the build's check 4), so `Jr`, which reads `post-nominal` until 4b, names `en`, the file of its `generation` reading. Only a `vocabulary` or `declared` reading is looked up; a shape, position, an initial or a patronymic names no file. A reading written `default = false` names none (`St` in a caller's `suffix_words`, held off by `cs.toml`), and neither does a word a caller's own `Lexicon` adds that no shipped file holds in that category (`Zzq` as a title), while one a file does hold names it (`Dean` added to `titles` names `en`), nor the Korean surnames that divide unspaced hangul, which `config/surnames.py` keeps outside the per-word files: `김` in `김민준` reads `name`, `vocabulary`, with no files. It names the default's files whatever `Lexicon` the parse used, the semantics decisions/A1's 2026-10-26 decline weighed against it: a file holds the reading or not, and which lexicon read it is the caller's own knowledge. Under a choice of languages (3b) a "live" reading becomes one the choice makes, and `languages_of` filtering on `on_by_default` is the place 3b changes. + 2. **How a word is looked up.** By the vocabulary's own form of the word (`_normalize`), then with every period removed, as a credential is matched (`M.A` in `John Smith (M.A)`) and as a word listed whole without them is (`Lt.Gen.` as `ltgen`); a credential split across a space whole, its first word joined to the second (`Ph.` `D.` as `phd`), the second taking the first's reading, past a nickname written between them too (`John Smith Ph. "Bob" D.`), where the second word read `credential`, `position` under 4a alone and now takes the first's `credential`, `vocabulary`: the one `Token.reading` this change moves, no role, tag or alternative moving with it; and a word joining listed words with punctuation by its parts, in order, a suffix's parts divided by any mark but the period first (`M.D./Ph.D.`, `MD;`), then any word's by every mark (`Lt.Gov.`), never at a combining mark, which `\W` would count (`प्रोफे.डॉ.` names `hi` and `mr` through `डॉ`), and only where the parse's own `Lexicon` does not list the word whole (in any form, its periods kept or removed): a caller's own `zzq-dr` title or `mac-van` particle was read whole, so it names none of `dr`'s or `van`'s files. An alternative names the files for any word its report names, each read in the field the alternative names. MEASURED 2026-10-29 on `bd02d68f` plus this change over the snapshot's 6,360 distinct names, the empty one among them (20,804 tokens): 3,935 readings are `vocabulary` or `declared` and 3,750 of them name a file; of the 185 that name none, 157 are Korean surnames split off by the list and 28 are declared names read into a field no file lists them in (`van` 17 as a family name, `le` 3, `san`, `la` and `fils` 2 each, `fils` once more and `von` once). Matching name rows by category rather than by field, as this entry's first draft did, 3,778 named a file, all of the 157 aside. By the normalized and period-free forms alone, without the split and the parts, the first draft counted 265 naming none: the 157, 42 words joining listed words, and 66 words of credentials split across a space, 40 by `split:credential` (`Ph.` `D.`) and 26 by `split:member` (`Pharm D`). Recompute: parse every `name` of `tools/differential/snapshot.jsonl` and count the tokens whose `reading.source` is `vocabulary` or `declared` by whether `reading.languages` is empty; for the second figure, look each up with `languages_of` by its normalized and period-free forms only, and sort the misses by the split tags on the word and the word before it. + 3. **The field.** `Reading.languages: tuple[str, ...] = ()`, `compare=False`, so two readings compare as in 2.x and every recorded alternative is untouched, and `metadata={ADDED_IN: "3.0.0"}`, `Reading` having shipped in 2.7: a `Reading` pickled by 2.7 or 2.8 loads with `()` (`tests/v2/data/parsedname_v28_alternative.pickle`, made by the released 2.8.0 wheel), and a renamed field or a shipped one missing beside it is still refused (`_REFUSED_SKEWS`). The constructor takes any iterable of str and refuses, with `TypeError`, a bare str, a mapping, a value that is not iterable and a non-str tag. The repr names the files where there are any (`Reading('particle' as 'family', 'vocabulary', in es fr it nl pt)`). The snapshot does not record readings, so nothing re-records; it is byte-identical. + 4. **Cost**, measured 2026-10-29, Python 3.11.17. Two memos keep the lookups off the frame count: `_languages`, an `lru_cache(maxsize=4096)` keyed on the word as written, the kind and the field (with `_part_languages`, its twin for the parts, asked only where `_languages` finds nothing), and an `lru_cache(maxsize=1024)` over `Reading` for the readings that name files, each a pure function of the shipped files, bounded because the caller's text keys them, and C-level on a hit, so a word read before costs no frame. `tools/perf/call_count.py`'s reference row moved from 311.00 on `bd02d68f` to 311.48, all of it the first reference parse of the 50 (335 frames against 311: the index import and the four first lookups; 331 and 311.40 before the review fixes); after it each parse counts 311.0 on both trees. `_CALL_BASELINE` (307) is not moved and the serial benchmark passes. Which parse loads the index is decided by the parse's own `Lexicon`, not the files: any `vocabulary` or `declared` reading looks its word up, so `김민준` and a caller's own title (`Zzq John Smith`) load it though no file holds either, while `John Smith` does not. In fresh processes with bytecode cached, as an installed wheel has it, the first parse to read a listed word (`Dr. Juan de la Vega III`, after a `John Smith`) takes 1.20 to 1.22 ms against 0.22 ms on `bd02d68f`, the medians of 21 processes in each of three runs: about 1.0 ms once per process, the index import 3a measured at 0.56 to 0.68 ms plus the first lookups and readings. The steady parse, over 4,000 names half of them listed-word names, takes 78.6 to 79.0 µs against 78.3 to 79.4 µs on `bd02d68f`, within the runs' noise. A rerun after the review fixes, on a quieter machine, measured the cached first parse at 0.78 ms (median of 15, twice). Without bytecode (`PYTHONDONTWRITEBYTECODE=1` and an empty `PYTHONPYCACHEPREFIX`, as in a container that never writes it) the same first parse takes 16.4 to 16.5 ms, the index compiled from source each process. Recompute: time the two parses in fresh processes and a `timeit` over the 4,000 names in each tree, with `PYTHONDONTWRITEBYTECODE` unset and the index's bytecode written, and the first parse again with it set and an empty cache prefix. 5. **Not done here: a listed suffix word's kind.** The index lookup this change makes does not settle 4b's `Jr` and `Sr` (`tests/v2/pipeline/test_assemble.py`'s strict expected failures stand): the kind a token takes is named by `_vocab.suffix_kind`, which also names `Ambiguity.alternative`, so reading the kind from the index at assemble alone would split the two, and reading it inside `suffix_kind` moves the alternatives and puts the index on the decision path. 4b decides it. diff --git a/docs/design/vocabulary-model-rfc.md b/docs/design/vocabulary-model-rfc.md index d3b4a194c..de7429b36 100644 --- a/docs/design/vocabulary-model-rfc.md +++ b/docs/design/vocabulary-model-rfc.md @@ -219,7 +219,7 @@ The 2 broken are pairs whose romanization opens with a listed title, which reads ### Where the provenance lives at runtime -The compiled module holds sets, and a set has forgotten which file each word came from. Phase 3 compiles a second module beside it (decided 2026-10-28): a **reading index**, one row per reading (its text, its file's tag, its kind and its behavioral properties), loaded once per process on its first lookup (`_reading_index.readings_of`) (decided 2026-10-29, replacing the 2026-10-28 decision to load it with `Lexicon.default()`). Phase 4's `Reading.languages` is filled from it at assemble (4a's follow-up, decisions/vocabulary-model's 2026-10-29 #754, 4a's follow-up entry), so the first parse in a process that reads a word from its `Lexicon`'s vocabulary loads it (a listed or declared word, a Korean surname dividing hangul or a caller's own word, whether or not a file holds it): every caller who parses such a name pays it once, about 1 ms on that parse, and a caller who only imports nameparser, or parses only words its `Lexicon` does not list, does not. Filling `reading.languages` on first access to that field instead was weighed and set aside: a `Reading` holds its kind, role and source but not its word, so a lazy field would have to carry the word as a hidden key on a frozen, slotted, pickled type, to save a cost the measurement below puts at about 2% of the import. The compiler moves from tools/vocab/build.py into the package so that the build and a pack share one implementation, and a test holds the invariant that compiling the index with no choice reproduces `nameparser/config/_compiled.py` exactly. The same index is what item 13 of decisions/vocabulary-model's 2026-10-23 #752 entry says `Token.reading` needs in 3.0; 3a built it (decisions/vocabulary-model, 2026-10-29), its import measured there at 0.56 to 0.68 ms. +The compiled module holds sets, and a set has forgotten which file each word came from. Phase 3 compiles a second module beside it (decided 2026-10-28): a **reading index**, one row per reading (its text, its file's tag, its kind and its behavioral properties), loaded once per process on its first lookup (`_reading_index.readings_of`, and `languages_of`, which a parse calls) (decided 2026-10-29, replacing the 2026-10-28 decision to load it with `Lexicon.default()`). Phase 4's `Reading.languages` is filled from it at assemble (4a's follow-up, decisions/vocabulary-model's 2026-10-29 #754, 4a's follow-up entry), so the first parse in a process that reads a word from its `Lexicon`'s vocabulary loads it (a listed or declared word, a Korean surname dividing hangul or a caller's own word, whether or not a file holds it): every caller who parses such a name pays it once, about 1 ms on that parse with bytecode cached and about 16 ms without it, and a caller who only imports nameparser, or parses only words its `Lexicon` does not list, does not. Filling `reading.languages` on first access to that field instead was weighed and set aside: a `Reading` holds its kind, role and source but not its word, so a lazy field would have to carry the word as a hidden key on a frozen, slotted, pickled type, to save a cost the measurement below puts at about 2% of the import. The compiler moves from tools/vocab/build.py into the package so that the build and a pack share one implementation, and a test holds the invariant that compiling the index with no choice reproduces `nameparser/config/_compiled.py` exactly. The same index is what item 13 of decisions/vocabulary-model's 2026-10-23 #752 entry says `Token.reading` needs in 3.0; 3a built it (decisions/vocabulary-model, 2026-10-29), its import measured there at 0.56 to 0.68 ms. Measured 2026-10-27 on `cfe8bf92`: the pool holds 2,140 readings over 1,797 words, 106 of them in more than one file. An index of their behavioral properties is about 93 KB of generated source. Imported with its bytecode cached, as an installed wheel has it, it takes about 0.45 ms (median of nine fresh processes, 2026-10-28 on `c8357e74`, Python 3.11.17), against about 20 ms for `import nameparser` cached the same way; with no cache it takes about 11 ms, the figure this section gave on 2026-10-27, measured in a shell that sets `PYTHONDONTWRITEBYTECODE`. That load is paid once per process, at the first lookup. Compiling the pool takes about 9 ms, paid once per choice and only by a caller who makes one. Reading the TOML instead costs about 39 ms. Recompute: the build's `read_dir`, `load` and `compile_pool`, timed; for the import, write the index as a module, `py_compile` it, and time its import in fresh processes beside a cached `import nameparser`. diff --git a/docs/release_log.rst b/docs/release_log.rst index 0d569f323..27ffc3a2d 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -7,7 +7,7 @@ Release Log **Additions** - **Add Token.reading, the reading the parse took for each word.** ``parse("John Smith PhD Jones").tokens[-1].reading`` is ``Reading('credential' as 'suffix', 'position')``, the reading 2.7's ``Ambiguity.alternative`` gave only for the other side of a fork, and ``ReadingKind`` gains ``patronymic``. The field is not compared, and a ``ParsedName`` pickled by 2.x loads with no readings. :issue:`754` - - **Add Reading.languages, the language files that list a word the way it was read.** ``parse("Dr. Juan de la Vega III").tokens[2].reading.languages`` is ``('es', 'fr', 'it', 'nl', 'pt')`` for the particle ``de``, and ``Ambiguity.alternative`` names its files too; a reading by shape or position names none. It is not compared, and a ``Reading`` pickled by 2.7 or 2.8 loads with none. The first parse in a process that meets a listed word loads the reading index, about 1 ms once. :issue:`754`, :issue:`753` + - **Add Reading.languages, the language files that list a word the way it was read.** ``parse("Dr. Juan de la Vega III").tokens[2].reading.languages`` is ``('es', 'fr', 'it', 'nl', 'pt')`` for the particle ``de``, and ``Ambiguity.alternative`` names its files too; a reading by shape or position names none. It is not compared, and a ``Reading`` pickled by 2.7 or 2.8 loads with none. The first parse in a process that meets a listed word loads the reading index once: about 1 ms, or about 16 ms where Python writes no bytecode cache. :issue:`754`, :issue:`753` **Breaking Changes** diff --git a/docs/usage.rst b/docs/usage.rst index eea49e461..7223c0063 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1213,7 +1213,8 @@ or position names none, and neither does a word from your own ``mul`` is the file of words shared across languages on purpose. The first parse in a process that meets a listed word loads the index of -those files, about a millisecond once. +those files once: about a millisecond, or about 16 ms where Python +writes no bytecode cache (``PYTHONDONTWRITEBYTECODE``). A token the parse did not read has no reading: one that :meth:`~nameparser.ParsedName.replace` or diff --git a/nameparser/_pipeline/AGENTS.md b/nameparser/_pipeline/AGENTS.md index ff9f8fc4a..5900df5a6 100644 --- a/nameparser/_pipeline/AGENTS.md +++ b/nameparser/_pipeline/AGENTS.md @@ -44,7 +44,7 @@ And work a report does must not walk the name: the given-name title test, asked ### The reading taken is read at assemble -`Token.reading` (3.0, #754's 4a) is not written by the deciding stages: assemble reads it from each token's final role and the tags the deciding stages left (classify's, the `joined` mark group and post_rules write, post_rules' unjoined marks), since a reading written where a word first takes its role would go stale at every later move (P1's fold, P6's attachment, the policy folds). The suffix and name halves ask `_vocab.suffix_kind` and `_vocab.name_source`, the predicates `suffix_reading` and `name_reading` use for the reading not taken, so a change to which reading a TAG offers changes both through one function there; the bound given name's source and the title, connective, particle and initial readings are assemble's own tests, kept out of the shared predicates because an arm added there moves every alternative they name (4a's first draft moved `Smith, John Abu`'s), the initial read off the `initial` tag where `name_reading` tests the shape. A new tag that says what a word was read as, or a decision whose outcome no tag and no role records, needs assemble taught too, with a row in `tests/v2/pipeline/test_assemble.py::_EXPECTED_READINGS`, which holds one row per branch assemble names a reading by; `test_every_parsed_token_has_a_reading_of_its_own_field` checks every case row has one, in its own field, and keeps it through `capitalized()`. decisions/vocabulary-model's 2026-10-29 #754, sub-phase 4a entry has which role reads as which reading. A reading's `languages` is assemble's too, looked up in the reading index by the word and the kind taken, never by a deciding stage: a new kind needs a row in `_assemble._CATEGORIES` naming the vocabulary categories that hold it, or it names no file, and a new way of writing a listed word (a split, a join) needs a row in `_EXPECTED_LANGUAGES`; its 2026-10-29 #754, 4a's follow-up entry has how a word is looked up. +`Token.reading` (3.0, #754's 4a) is not written by the deciding stages: assemble reads it from each token's final role and the tags the deciding stages left (classify's, the `joined` mark group and post_rules write, post_rules' unjoined marks), since a reading written where a word first takes its role would go stale at every later move (P1's fold, P6's attachment, the policy folds). The suffix and name halves ask `_vocab.suffix_kind` and `_vocab.name_source`, the predicates `suffix_reading` and `name_reading` use for the reading not taken, so a change to which reading a TAG offers changes both through one function there; the bound given name's source and the title, connective, particle and initial readings are assemble's own tests, kept out of the shared predicates because an arm added there moves every alternative they name (4a's first draft moved `Smith, John Abu`'s), the initial read off the `initial` tag where `name_reading` tests the shape. A new tag that says what a word was read as, or a decision whose outcome no tag and no role records, needs assemble taught too, with a row in `tests/v2/pipeline/test_assemble.py::_EXPECTED_READINGS`, which holds one row per branch assemble names a reading by; `test_every_parsed_token_has_a_reading_of_its_own_field` checks every case row has one, in its own field, and keeps it through `capitalized()`. decisions/vocabulary-model's 2026-10-29 #754, sub-phase 4a entry has which role reads as which reading. A reading's `languages` is assemble's too, looked up in the reading index by the word, the kind taken and, for a name, the field, never by a deciding stage: a new kind needs a row in `_assemble._KINDS` naming the vocabulary kinds that hold it, or it names no file, a word joining listed words is read by its parts only where the parse's own `Lexicon` does not list it whole (`_files`), and a new way of writing a listed word (a split, a join) needs a row in `_EXPECTED_LANGUAGES`; its 2026-10-29 #754, 4a's follow-up entry has how a word is looked up. ### Testing an emitter diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py index 868fee03d..1199363aa 100644 --- a/nameparser/_pipeline/_assemble.py +++ b/nameparser/_pipeline/_assemble.py @@ -35,7 +35,7 @@ import unicodedata from types import MappingProxyType -from nameparser._lexicon import Lexicon, _normalize +from nameparser._lexicon import _VOCAB_FIELDS, Lexicon, _normalize from nameparser._pipeline._state import ( AGNOMEN_TAG, AMBIGUOUS_ACRONYM_TAG, AMBIGUOUS_WORD_TAG, FIELD_READING, NAME_ROLES, SHAPE_ACRONYM_TAG, SPLIT_CREDENTIAL_TAG, SPLIT_MEMBER_TAG, @@ -45,7 +45,7 @@ from nameparser._pipeline._vocab import ( _PERIOD_ABBREV, name_source, suffix_kind, ) -from nameparser._reading_index import languages_of +from nameparser._reading_index import CATEGORY, languages_of from nameparser._types import ( FOLDED_TAG, GIVEN_FIRST, UNJOINED_CONJUNCTION_TAG, UNJOINED_TAG, Ambiguity, AmbiguityKind, ParsedName, Reading, ReadingKind, @@ -60,57 +60,91 @@ (kind, role, source): Reading(kind, role, source) for kind in ReadingKind for role in Role for source in ReadingSource}) -#: The vocabulary categories (_reading_index.CATEGORY's values) whose -#: rows hold a reading of each kind. A suffix reading takes the word's +def _kinds_of(*categories: str) -> frozenset[str]: + return frozenset(k for k, c in CATEGORY.items() if c in categories) + + +#: The vocabulary kinds (_reading_index.CATEGORY's keys) whose rows +#: hold a reading of each kind. A suffix reading takes the word's #: suffix rows whatever their finer kind, since a word has one suffix -#: kind (the build's check 4) and the parse does not yet read it (4b); -#: a name, its name and bound given name rows. An initial and a -#: patronymic are read by shape or rule, never from a file. -_CATEGORIES: MappingProxyType[ReadingKind, frozenset[str]] = MappingProxyType({ - ReadingKind.TITLE: frozenset({"title"}), - ReadingKind.PARTICLE: frozenset({"particle"}), - ReadingKind.CONJUNCTION: frozenset({"conjunction"}), - ReadingKind.CREDENTIAL: frozenset({"suffix"}), - ReadingKind.GENERATION: frozenset({"suffix"}), - ReadingKind.AGNOMEN: frozenset({"suffix"}), - ReadingKind.POST_NOMINAL: frozenset({"suffix"}), - ReadingKind.NAME: frozenset({"name", "bound_given"}), +#: kind (the build's check 4) and the parse does not yet read it (4b). +#: A name reading's rows depend on its field (`_NAME_KINDS`). An +#: initial and a patronymic are read by shape or rule, never from a +#: file. +_SUFFIX_KINDS = _kinds_of("suffix") +_KINDS: MappingProxyType[ReadingKind, frozenset[str]] = MappingProxyType({ + ReadingKind.TITLE: _kinds_of("title"), + ReadingKind.PARTICLE: _kinds_of("particle"), + ReadingKind.CONJUNCTION: _kinds_of("conjunction"), + ReadingKind.CREDENTIAL: _SUFFIX_KINDS, + ReadingKind.GENERATION: _SUFFIX_KINDS, + ReadingKind.AGNOMEN: _SUFFIX_KINDS, + ReadingKind.POST_NOMINAL: _SUFFIX_KINDS, +}) + +#: A name reading's rows by the field it lands in: a family word the +#: files listing it as a family name, a given or middle word those +#: listing it as a given name ('Ma' as a family name names `en`, not +#: `fil`, which lists it only as given), and a bound given name +#: (rules/P5) in any field, its rows being the reason it was read so; +#: a name in another field (a nickname) any name rows. +_GIVEN_KINDS = frozenset({"given", "bound_given"}) +_FAMILY_KINDS = frozenset({"family", "bound_given"}) +_NAME_KINDS: MappingProxyType[Role, frozenset[str]] = MappingProxyType({ + Role.GIVEN: _GIVEN_KINDS, Role.MIDDLE: _GIVEN_KINDS, + Role.FAMILY: _FAMILY_KINDS, Role.MAIDEN: _FAMILY_KINDS, }) +_ANY_NAME_KINDS = _kinds_of("name", "bound_given") #: The sources a file can hold: a listed word, and a name the #: vocabulary declares beside another reading. _LISTED = frozenset({ReadingSource.VOCABULARY, ReadingSource.DECLARED}) -# Both memos are C-level on a hit, so a word read before costs no -# frame (decisions/parse-cost), and both are pure functions of the +def _kinds(kind: ReadingKind, role: Role) -> frozenset[str] | None: + return (_NAME_KINDS.get(role, _ANY_NAME_KINDS) + if kind is ReadingKind.NAME else _KINDS.get(kind)) + + +# Both lookups are memos, C-level on a hit, so a word read before costs +# no frame (decisions/parse-cost), and both are pure functions of the # shipped files, bounded since what keys them comes from the caller's -# text: `_languages` of a word as written, `_reading` of the readings -# that name files. +# text; so is `_reading`, of the readings that name files. @functools.lru_cache(maxsize=4096) -def _languages(text: str, kind: ReadingKind) -> tuple[str, ...]: - """The files holding the `kind` reading of `text` as written, - Reading.languages: by the vocabulary's own form of the word; a - suffix also with every period removed, the form a credential is - matched by ('M.A' in 'John Smith (M.A)'); and a word joining - listed words with punctuation ('Lt.Gov.', 'MD/PhD', 'MD;') by - the files of its parts, in order.""" - cats = _CATEGORIES.get(kind) - if cats is None: +def _languages(text: str, kind: ReadingKind, role: Role) -> tuple[str, ...]: + """The files holding the `kind` reading of `text` as written, in + `role`, Reading.languages: by the vocabulary's own form of the + word, and then with every period removed, the form a credential is + matched by ('M.A' in 'John Smith (M.A)') and a word listed whole + without them ('Lt.Gen.' as 'ltgen').""" + kinds = _kinds(kind, role) + if kinds is None: return () n = _normalize(text) - acronym = "suffix" in cats - found = languages_of(n, cats) - if not found and acronym and "." in n: - found = languages_of(n.replace(".", ""), cats) - if not found and acronym: - # its parts by any other mark, each an acronym ('M.D./Ph.D.') - found = _by_parts(n, cats, ".") - return found or _by_parts(n, cats, "") - - -def _by_parts(n: str, cats: frozenset[str], keep: str) -> tuple[str, ...]: - """The files of the parts of `n`, in order: divided by punctuation, + found = languages_of(n, kinds) + if not found and "." in n: + found = languages_of(n.replace(".", ""), kinds) + return found + + +@functools.lru_cache(maxsize=4096) +def _part_languages(text: str, kind: ReadingKind, + role: Role) -> tuple[str, ...]: + """The files of the listed words `text` joins with punctuation + ('Lt.Gov.', 'MD/PhD', 'MD;'), sorted: a suffix's parts divided by + any mark but the period first, each an acronym ('M.D./Ph.D.'), + then any word's by every mark. Read only for a word the parse's + own lexicon does not list whole (`_files`).""" + kinds = _kinds(kind, role) + if kinds is None: + return () + n = _normalize(text) + found = _by_parts(n, kinds, ".") if kinds is _SUFFIX_KINDS else () + return found or _by_parts(n, kinds, "") + + +def _by_parts(n: str, kinds: frozenset[str], keep: str) -> tuple[str, ...]: + """The files of the parts of `n`, sorted: divided by punctuation, symbols and spaces, but for the marks in `keep`, which are removed from each part instead; never by a combining mark, which a `[^\\w]` would count ('डॉ' in 'प्रोफे.डॉ.').""" @@ -120,7 +154,26 @@ def _by_parts(n: str, cats: frozenset[str], keep: str) -> tuple[str, ...]: return tuple(sorted({ tag for part in parts for tag in languages_of(part.replace(keep, "") if keep else part, - cats)})) + kinds)})) + + +def _files(text: str, kind: ReadingKind, role: Role, + lexicon: Lexicon) -> tuple[str, ...]: + """Reading.languages for a word whose whole form no file holds + (`_languages` empty): the files of the listed words it joins, save + where the parse's own lexicon lists it whole, so that the parse + read it whole and not by its parts ('zzq-dr' a caller added as a + title names no file, though 'dr' is in `mul`). Asked only after + `_languages` comes back empty, so a word a file holds pays nothing.""" + found = _part_languages(text, kind, role) + if found: + n = _normalize(text) + bare = n.replace(".", "") + for name in _VOCAB_FIELDS: + field = getattr(lexicon, name) + if n in field or bare in field: + return () + return found def _split_partner(tokens: tuple[WorkToken, ...], dropped: set[int], @@ -158,8 +211,8 @@ def _split_partner(tokens: tuple[WorkToken, ...], dropped: set[int], def _alternative(pending: PendingAmbiguity, materialized: tuple[Token, ...], - final: dict[int, Token], - middle_as_family: bool) -> Reading | None: + final: dict[int, Token], middle_as_family: bool, + lexicon: Lexicon) -> Reading | None: """The public Reading of a report's PendingReading, its field named from the final roles where the emitter deferred it (O5's other primary field) and folded as `middle_as_family` folds every middle @@ -188,7 +241,8 @@ def _alternative(pending: PendingAmbiguity, materialized: tuple[Token, ...], if alt.source in _LISTED: # a loop, not a generator, which would resume a frame per word for t in materialized: - found.update(_languages(t.text, alt.kind)) + found.update(_languages(t.text, alt.kind, role) + or _files(t.text, alt.kind, role, lexicon)) return Reading(alt.kind, role, alt.source, tuple(sorted(found))) @@ -285,7 +339,9 @@ def assemble(state: ParseState) -> ParsedName: if prev_split else t.text) reading = (_reading(kind, role, source, langs) if source in _LISTED - and (langs := _languages(written, kind)) + and (langs := _languages(written, kind, role) + or _files(written, kind, role, + state.lexicon)) else _READINGS[kind, role, source]) before = reading else: @@ -350,7 +406,8 @@ def assemble(state: ParseState) -> ParsedName: kind, source = ReadingKind.NAME, name_source(tags) reading = (_reading(kind, role, source, langs) if source in _LISTED - and (langs := _languages(t.text, kind)) + and (langs := _languages(t.text, kind, role) + or _files(t.text, kind, role, state.lexicon)) else _READINGS[kind, role, source]) final[i] = Token(t.text, t.span, role, tags, reading) # No alphanumeric character among the SURVIVING tokens means no @@ -460,7 +517,7 @@ def assemble(state: ParseState) -> ParsedName: ambiguities.append(Ambiguity( pending.kind, detail, materialized, _alternative(pending, materialized, final, - state.policy.middle_as_family))) + state.policy.middle_as_family, state.lexicon))) return ParsedName(original=state.original, tokens=tuple(final.values()), ambiguities=tuple(ambiguities), diff --git a/nameparser/_reading_index.py b/nameparser/_reading_index.py index 0f70b537e..0e046fd98 100644 --- a/nameparser/_reading_index.py +++ b/nameparser/_reading_index.py @@ -238,17 +238,16 @@ def readings_of(text: str) -> tuple[IndexReading, ...]: return tuple(from_row(text, row) for row in _by_text().get(text, ())) -def languages_of(text: str, categories: frozenset[str]) -> tuple[str, ...]: +def languages_of(text: str, kinds: frozenset[str]) -> tuple[str, ...]: """The tags of the files holding a reading of `text`, on by - default, whose kind falls in one of `categories` (values of - :data:`CATEGORY`), each once, in index order; empty for a word no - file holds in them. `text` is normalized as :func:`readings_of` - takes it. What assemble fills ``Reading.languages`` with - (decisions/vocabulary-model, 2026-10-29); its first call in a - process loads the index.""" + default, of one of `kinds` (keys of :data:`CATEGORY`), each once, + in index order; empty for a word no file holds as one of them. + `text` is normalized as :func:`readings_of` takes it. What + assemble fills ``Reading.languages`` with (decisions/vocabulary- + model, 2026-10-29); its first call in a process loads the index.""" out: list[str] = [] for tag, kind, pairs in _by_text().get(text, ()): - if (CATEGORY[kind] in categories and tag not in out + if (kind in kinds and tag not in out and ("default", False) not in pairs): out.append(tag) return tuple(out) diff --git a/nameparser/_types.py b/nameparser/_types.py index 94d433b2e..f9ce29ced 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -952,7 +952,11 @@ class Reading: #: The BCP 47 tags of the shipped vocabulary files that hold this #: reading of the word, on by default (``("es", "fr", "it", "nl", #: "pt")`` for the particle ``de``, ``("mul",)`` for the title - #: ``Dr.``), sorted. Empty for a reading no file holds: a shape, + #: ``Dr.``), sorted; a name reading takes the files listing the + #: word as a name of its field, given or family (``Ma`` read as a + #: family name names ``en``, which lists it so, and not ``fil``, + #: which lists it as a given name). Empty for a reading no file + #: holds: a shape, #: position alone, a word a caller's own ``Lexicon`` adds that no #: file holds in that category, or a shipped list kept outside the #: per-word files (the Korean surnames that divide ``김민준``). @@ -971,12 +975,17 @@ def __post_init__(self) -> None: # the members the parser passes need no coercion, and a report # pays no frame for it (decisions/parse-cost) if type(self.languages) is not tuple: - if isinstance(self.languages, str): + langs = self.languages + if isinstance(langs, str): raise TypeError( f"Reading.languages must be an iterable of BCP 47 " - f"tags, got the str {self.languages!r}; write " - f"({self.languages!r},)") - object.__setattr__(self, "languages", tuple(self.languages)) + f"tags, got the str {langs!r}; write ({langs!r},)") + if isinstance(langs, Mapping) or not isinstance(langs, Iterable): + raise TypeError( + f"Reading.languages must be an iterable of BCP 47 " + f"tags as str, got {langs!r}; pass a tuple or list of " + f"tags, such as ('en',)") + object.__setattr__(self, "languages", tuple(langs)) for tag in self.languages: if not isinstance(tag, str): raise TypeError( diff --git a/tests/v2/pipeline/test_assemble.py b/tests/v2/pipeline/test_assemble.py index 3a02264d0..19564dde6 100644 --- a/tests/v2/pipeline/test_assemble.py +++ b/tests/v2/pipeline/test_assemble.py @@ -441,6 +441,10 @@ def test_a_suffix_word_reads_as_its_vocabulary_kind( # a name the vocabulary declares, and a bound given name ("van Gogh", "van", ("nl",)), ("Chas. Smith", "Chas.", ("en",)), + # a name reading names the files listing the word as a name of its + # own field: `ma` is family in en, given in en and fil + ("Long Ma", "Ma", ("en",)), + ("Ma Long", "Ma", ("en", "fil")), ("Abu Bakar Smith", "Abu", ("ar",)), # an acronym by its periods removed, and a credential split across # a space, listed whole, each of its words naming the files @@ -457,6 +461,9 @@ def test_a_suffix_word_reads_as_its_vocabulary_kind( # combining mark ("John Smith, M.D./Ph.D.", "M.D./Ph.D.", ("en",)), ("Lt.Gov. John Smith", "Lt.Gov.", ("en",)), + # a word the lexicon lists whole without its periods is read whole, + # not by its parts + ("Lt.Gen. John Smith", "Lt.Gen.", ("en",)), ("प्रोफे.डॉ. अदा शर्मा", "प्रोफे.डॉ.", ("hi", "mr")), # a union over parts is sorted, as a word's own files are ("John Smith, MS-V", "MS-V", ("en", "mul")), @@ -480,6 +487,23 @@ def test_a_reading_names_the_files_holding_it( assert tok.reading.languages == languages +@pytest.mark.parametrize("field,word,text", [ + ("titles", "zzq-dr", "Zzq-Dr John Smith"), + ("particles", "mac-van", "John mac-van Smith"), +]) +def test_a_callers_whole_word_names_no_file_of_its_parts( + field: str, word: str, text: str) -> None: + # the parse read it whole, so the files of `dr` or `van` are not + # its reading's; the default's own joined words still name theirs + # (`Lt.Gov.` in _EXPECTED_LANGUAGES) + lex = Lexicon.default().add(**{field: {word}}) + (tok,) = [t for t in Parser(lexicon=lex).parse(text).tokens + if t.text.lower() == word] + assert tok.reading is not None + assert tok.reading.source is ReadingSource.VOCABULARY + assert tok.reading.languages == () + + def test_a_reading_from_a_callers_own_vocabulary_names_no_file() -> None: # a word no file holds, and one a file holds only off by default # (`st`, a Czech generation written `default = false`) @@ -495,12 +519,13 @@ def test_a_reading_from_a_callers_own_vocabulary_names_no_file() -> None: def test_an_alternative_names_the_files_holding_it() -> None: (amb,) = parse("JOHN SMITH MA").ambiguities assert amb.alternative is not None - # `ma` as a name: given and family in en, given in fil - assert amb.alternative.languages == ("en", "fil") + # `ma` as a family name: en lists it so, fil only as a given name + assert amb.alternative.languages == ("en",) (amb,) = parse("van Gogh").ambiguities assert amb.alternative is not None assert amb.alternative.languages == ("nl",) - # over the report's words, sorted ('van' nl, 'DO' en) + # over the report's words, sorted ('van' and 'DO' as given names, + # nl and en) (amb,) = parse("Doe, John van DO").ambiguities assert amb.alternative is not None assert amb.alternative.languages == ("en", "nl") diff --git a/tests/v2/test_reprs.py b/tests/v2/test_reprs.py index c705762ea..f600d4134 100644 --- a/tests/v2/test_reprs.py +++ b/tests/v2/test_reprs.py @@ -32,7 +32,7 @@ def test_ambiguity_repr_shows_kind_and_token_texts() -> None: def test_reading_repr_shows_kind_field_and_source() -> None: alt = parse("JOHN SMITH MA").ambiguities[0].alternative - assert repr(alt) == "Reading('name' as 'family', 'declared', in en fil)" + assert repr(alt) == "Reading('name' as 'family', 'declared', in en)" # the files are left out where none holds the reading assert repr(Reading("name", "family", "position")) == ( # type: ignore[arg-type] "Reading('name' as 'family', 'position')") diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index c4b949465..29f9d4267 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -528,10 +528,20 @@ def test_reading_languages_are_a_tuple_of_tags_and_are_not_compared() -> None: assert bare.languages == () assert held.languages == ("es", "fr") assert held == bare and hash(held) == hash(bare) - with pytest.raises(TypeError, match=r"got the str 'es'; write \('es',\)"): - Reading("particle", "family", "vocabulary", "es") # type: ignore[arg-type] - with pytest.raises(TypeError, match="as str, got 1"): - Reading("particle", "family", "vocabulary", ("es", 1)) # type: ignore[arg-type] + + + +@pytest.mark.parametrize("bad,match", [ + ("es", r"got the str 'es'; write \('es',\)"), + ({"es": 1}, r"got \{'es': 1\}; pass a tuple or list of tags"), + (5, r"got 5; pass a tuple or list of tags, such as \('en',\)"), + (("es", 1), r"as str, got 1 in \('es', 1\)"), +], ids=["bare-str", "mapping", "not-iterable", "non-str-tag"]) +def test_reading_languages_refuses_what_is_not_tags( + bad: object, match: str) -> None: + from nameparser import Reading + with pytest.raises(TypeError, match=match): + Reading("particle", "family", "vocabulary", bad) # type: ignore[arg-type] def test_ambiguity_alternative_is_a_reading_and_is_not_compared() -> None: @@ -716,7 +726,7 @@ def test_a_reading_pickled_by_2_8_0_loads_without_languages() -> None: (old,), (new,) = loaded.ambiguities, now.ambiguities assert old.alternative is not None and new.alternative is not None assert old.alternative.languages == () - assert new.alternative.languages == ("en", "fil") + assert new.alternative.languages == ("en",) assert old.alternative == new.alternative again = pickle.loads(pickle.dumps(now)) assert [t.reading.languages for t in again.tokens if t.reading] == [ From 4c8db54f2327f81b55ccf194b31182f2e3e19dec Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 29 Oct 2026 14:51:51 -0700 Subject: [PATCH 6/7] Keep the lexicon check off the frame count; read a folded middle as a middle (#754) Answers the review of 2aab5468. The check that the parse's own Lexicon lists a word whole ran as a frame on every parse for each listed word whose whole form no file holds ('van' as a family name, 'Fils' as a middle name); it now runs only where some file holds the word's parts, behind two memos, so those words pay no frame again and only a word read by its parts pays one. A middle word that middle_as_family folds into the family names the files listing it as a given name, as without the policy, and an alternative's files are read before that fold. Test rows now hold the maiden, middle and bound-given arms of the field lookup and the period-free arm of the lexicon check; each fails under its own mutation. Co-Authored-By: Claude Opus 5.5 --- docs/design/decisions/vocabulary-model.md | 4 +- nameparser/_pipeline/_assemble.py | 79 ++++++++++++++--------- nameparser/_types.py | 8 ++- tests/v2/pipeline/test_assemble.py | 43 ++++++++++-- 4 files changed, 91 insertions(+), 43 deletions(-) diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index 5b9ceb561..6d807c72f 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -291,8 +291,8 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) 5. **Left open.** A listed suffix word that the compiled sets do not split by kind reads `post-nominal`, so `Jr`, `Sr` and `Esq.` read alike: the TOML gives `jr` a `generation` reading, but `SUFFIX_WORDS` holds both kinds and classify tags them alike, so only a vocabulary lookup by kind can tell them apart: 3a's reading index, a new `Lexicon` field being set aside (the 2026-10-28 `Lexicon` surface entry makes the compiled sets internal, and `Lexicon`'s own pickle guard refuses an added field). It is the predicate's own answer, which `Ambiguity.alternative` has given since 2.7 (the snapshot's one post-nominal alternative is `Rinpoche`'s), and a kind left unset would have meant a parsed token with no reading or a kind for "not yet known", so the reading taken agrees with it until 4b, whose `generation` view needs the split, reads a listed word's kind from 3a's index; `tests/v2/pipeline/test_assemble.py::test_a_suffix_word_reads_as_its_vocabulary_kind` holds `Jr` and `Sr` as strict expected failures until then. `ReadingKind.PATRONYMIC` exists and no reading takes it until 4c, whose rules write it. This amends the 2026-10-28 #754 entry's item 9: 4b needs 3a for that kind. - **2026-10-29 #754, 4a's follow-up — `Reading.languages` IS FILLED AT ASSEMBLE FROM 3a's READING INDEX, FOR THE READING A TOKEN TOOK AND FOR A REPORT'S ALTERNATIVE; NO PARSE MOVES.** Completes 4a (the 2026-10-29 #754, sub-phase 4a entry) with the field item 2 of the 2026-10-28 #754 entry names. Until it, no parse read the index (the 2026-10-29 #753, phase 3a entry's load bullet); since it, the first parse that reads a word from the vocabulary of its own `Lexicon` loads it (item 4). SUPERSEDES: decisions/A1's 2026-10-26 #752 (phase 2m) entry's decline of the reading's `languages`, for 3.0; its reason, that the index describes the default and not the `Lexicon` a parse used, is answered by item 1's semantics rather than removed. - 1. **What it holds.** The BCP 47 tags of the shipped files holding the reading taken, on by default, each once, sorted (a word joined from parts and an alternative over several words sorting the union), read through `_reading_index.languages_of(text, kinds)`, which walks the index rows without building an `IndexReading`. A reading matches the rows of every vocabulary kind of its category: a `title`, `particle` or `conjunction` reading its category's rows; a `name` reading the rows listing the word as a name of the field it lands in, `family` for a family or maiden word and `given` for a given or middle one, with the `bound_given` rows in either (a nickname takes any name rows), so `Ma` read as a family name (`Long Ma`) names `en` and not `fil`, which lists it only as a given name, and a declared name read into a field no file lists it in names none (`van` as a family name, which `nl` lists as given); and every suffix kind the word's suffix rows, a word having one suffix kind (the build's check 4), so `Jr`, which reads `post-nominal` until 4b, names `en`, the file of its `generation` reading. Only a `vocabulary` or `declared` reading is looked up; a shape, position, an initial or a patronymic names no file. A reading written `default = false` names none (`St` in a caller's `suffix_words`, held off by `cs.toml`), and neither does a word a caller's own `Lexicon` adds that no shipped file holds in that category (`Zzq` as a title), while one a file does hold names it (`Dean` added to `titles` names `en`), nor the Korean surnames that divide unspaced hangul, which `config/surnames.py` keeps outside the per-word files: `김` in `김민준` reads `name`, `vocabulary`, with no files. It names the default's files whatever `Lexicon` the parse used, the semantics decisions/A1's 2026-10-26 decline weighed against it: a file holds the reading or not, and which lexicon read it is the caller's own knowledge. Under a choice of languages (3b) a "live" reading becomes one the choice makes, and `languages_of` filtering on `on_by_default` is the place 3b changes. + 1. **What it holds.** The BCP 47 tags of the shipped files holding the reading taken, on by default, each once, sorted (a word joined from parts and an alternative over several words sorting the union), read through `_reading_index.languages_of(text, kinds)`, which walks the index rows without building an `IndexReading`. A reading matches the rows of every vocabulary kind of its category: a `title`, `particle` or `conjunction` reading its category's rows; a `name` reading the rows listing the word as a name of the field it lands in, `family` for a family or maiden word and `given` for a given or middle one, with the `bound_given` rows in either (a nickname takes any name rows), and a middle word `middle_as_family` folds into the family by the middle it stands as, the policy choosing the reported field and not the files (`John Ma Smith` under it names `en fil`, as without it), so `Ma` read as a family name (`Long Ma`) names `en` and not `fil`, which lists it only as a given name, and a declared name read into a field no file lists it in names none (`van` as a family name, which `nl` lists as given); and every suffix kind the word's suffix rows, a word having one suffix kind (the build's check 4), so `Jr`, which reads `post-nominal` until 4b, names `en`, the file of its `generation` reading. Only a `vocabulary` or `declared` reading is looked up; a shape, position, an initial or a patronymic names no file. A reading written `default = false` names none (`St` in a caller's `suffix_words`, held off by `cs.toml`), and neither does a word a caller's own `Lexicon` adds that no shipped file holds in that category (`Zzq` as a title), while one a file does hold names it (`Dean` added to `titles` names `en`), nor the Korean surnames that divide unspaced hangul, which `config/surnames.py` keeps outside the per-word files: `김` in `김민준` reads `name`, `vocabulary`, with no files. It names the default's files whatever `Lexicon` the parse used, the semantics decisions/A1's 2026-10-26 decline weighed against it: a file holds the reading or not, and which lexicon read it is the caller's own knowledge. Under a choice of languages (3b) a "live" reading becomes one the choice makes, and `languages_of` filtering on `on_by_default` is the place 3b changes. 2. **How a word is looked up.** By the vocabulary's own form of the word (`_normalize`), then with every period removed, as a credential is matched (`M.A` in `John Smith (M.A)`) and as a word listed whole without them is (`Lt.Gen.` as `ltgen`); a credential split across a space whole, its first word joined to the second (`Ph.` `D.` as `phd`), the second taking the first's reading, past a nickname written between them too (`John Smith Ph. "Bob" D.`), where the second word read `credential`, `position` under 4a alone and now takes the first's `credential`, `vocabulary`: the one `Token.reading` this change moves, no role, tag or alternative moving with it; and a word joining listed words with punctuation by its parts, in order, a suffix's parts divided by any mark but the period first (`M.D./Ph.D.`, `MD;`), then any word's by every mark (`Lt.Gov.`), never at a combining mark, which `\W` would count (`प्रोफे.डॉ.` names `hi` and `mr` through `डॉ`), and only where the parse's own `Lexicon` does not list the word whole (in any form, its periods kept or removed): a caller's own `zzq-dr` title or `mac-van` particle was read whole, so it names none of `dr`'s or `van`'s files. An alternative names the files for any word its report names, each read in the field the alternative names. MEASURED 2026-10-29 on `bd02d68f` plus this change over the snapshot's 6,360 distinct names, the empty one among them (20,804 tokens): 3,935 readings are `vocabulary` or `declared` and 3,750 of them name a file; of the 185 that name none, 157 are Korean surnames split off by the list and 28 are declared names read into a field no file lists them in (`van` 17 as a family name, `le` 3, `san`, `la` and `fils` 2 each, `fils` once more and `von` once). Matching name rows by category rather than by field, as this entry's first draft did, 3,778 named a file, all of the 157 aside. By the normalized and period-free forms alone, without the split and the parts, the first draft counted 265 naming none: the 157, 42 words joining listed words, and 66 words of credentials split across a space, 40 by `split:credential` (`Ph.` `D.`) and 26 by `split:member` (`Pharm D`). Recompute: parse every `name` of `tools/differential/snapshot.jsonl` and count the tokens whose `reading.source` is `vocabulary` or `declared` by whether `reading.languages` is empty; for the second figure, look each up with `languages_of` by its normalized and period-free forms only, and sort the misses by the split tags on the word and the word before it. 3. **The field.** `Reading.languages: tuple[str, ...] = ()`, `compare=False`, so two readings compare as in 2.x and every recorded alternative is untouched, and `metadata={ADDED_IN: "3.0.0"}`, `Reading` having shipped in 2.7: a `Reading` pickled by 2.7 or 2.8 loads with `()` (`tests/v2/data/parsedname_v28_alternative.pickle`, made by the released 2.8.0 wheel), and a renamed field or a shipped one missing beside it is still refused (`_REFUSED_SKEWS`). The constructor takes any iterable of str and refuses, with `TypeError`, a bare str, a mapping, a value that is not iterable and a non-str tag. The repr names the files where there are any (`Reading('particle' as 'family', 'vocabulary', in es fr it nl pt)`). The snapshot does not record readings, so nothing re-records; it is byte-identical. - 4. **Cost**, measured 2026-10-29, Python 3.11.17. Two memos keep the lookups off the frame count: `_languages`, an `lru_cache(maxsize=4096)` keyed on the word as written, the kind and the field (with `_part_languages`, its twin for the parts, asked only where `_languages` finds nothing), and an `lru_cache(maxsize=1024)` over `Reading` for the readings that name files, each a pure function of the shipped files, bounded because the caller's text keys them, and C-level on a hit, so a word read before costs no frame. `tools/perf/call_count.py`'s reference row moved from 311.00 on `bd02d68f` to 311.48, all of it the first reference parse of the 50 (335 frames against 311: the index import and the four first lookups; 331 and 311.40 before the review fixes); after it each parse counts 311.0 on both trees. `_CALL_BASELINE` (307) is not moved and the serial benchmark passes. Which parse loads the index is decided by the parse's own `Lexicon`, not the files: any `vocabulary` or `declared` reading looks its word up, so `김민준` and a caller's own title (`Zzq John Smith`) load it though no file holds either, while `John Smith` does not. In fresh processes with bytecode cached, as an installed wheel has it, the first parse to read a listed word (`Dr. Juan de la Vega III`, after a `John Smith`) takes 1.20 to 1.22 ms against 0.22 ms on `bd02d68f`, the medians of 21 processes in each of three runs: about 1.0 ms once per process, the index import 3a measured at 0.56 to 0.68 ms plus the first lookups and readings. The steady parse, over 4,000 names half of them listed-word names, takes 78.6 to 79.0 µs against 78.3 to 79.4 µs on `bd02d68f`, within the runs' noise. A rerun after the review fixes, on a quieter machine, measured the cached first parse at 0.78 ms (median of 15, twice). Without bytecode (`PYTHONDONTWRITEBYTECODE=1` and an empty `PYTHONPYCACHEPREFIX`, as in a container that never writes it) the same first parse takes 16.4 to 16.5 ms, the index compiled from source each process. Recompute: time the two parses in fresh processes and a `timeit` over the 4,000 names in each tree, with `PYTHONDONTWRITEBYTECODE` unset and the index's bytecode written, and the first parse again with it set and an empty cache prefix. + 4. **Cost**, measured 2026-10-29, Python 3.11.17. Two memos keep the lookups off the frame count: `_languages`, an `lru_cache(maxsize=4096)` keyed on the word as written, the kind and the field (with `_part_languages`, its twin for the parts, asked only where `_languages` finds nothing, and `_forms`, a word's two normalized forms), and an `lru_cache(maxsize=1024)` over `Reading` for the readings that name files, each a pure function of the shipped files, bounded because the caller's text keys them, and C-level on a hit, so a word read before costs no frame. The one frame left is `_listed_whole`, the lexicon check on a word read by its parts, which looks the word up in the parse's own `Lexicon` and so cannot be memoized on the text: `Lt.Gov. John Smith` and `John Smith, M.D./Ph.D.` count one frame more per parse than on `dce56273`, where the parts were read inside the memo with no lexicon check, while a word no part of which a file holds (`van` as a family name, `김`) pays none. `tools/perf/call_count.py`'s reference row moved from 311.00 on `bd02d68f` to 311.48, all of it the first reference parse of the 50 (335 frames against 311: the index import and the four first lookups; 331 and 311.40 before the review fixes); after it each parse counts 311.0 on both trees. `_CALL_BASELINE` (307) is not moved and the serial benchmark passes. Which parse loads the index is decided by the parse's own `Lexicon`, not the files: any `vocabulary` or `declared` reading looks its word up, so `김민준` and a caller's own title (`Zzq John Smith`) load it though no file holds either, while `John Smith` does not. In fresh processes with bytecode cached, as an installed wheel has it, the first parse to read a listed word (`Dr. Juan de la Vega III`, after a `John Smith`) takes 1.20 to 1.22 ms against 0.22 ms on `bd02d68f`, the medians of 21 processes in each of three runs: about 1.0 ms once per process, the index import 3a measured at 0.56 to 0.68 ms plus the first lookups and readings. The steady parse, over 4,000 names half of them listed-word names, takes 78.6 to 79.0 µs against 78.3 to 79.4 µs on `bd02d68f`, within the runs' noise. A rerun after the review fixes, on a quieter machine, measured the cached first parse at 0.78 ms (median of 15, twice). Without bytecode (`PYTHONDONTWRITEBYTECODE=1` and an empty `PYTHONPYCACHEPREFIX`, as in a container that never writes it) the same first parse takes 16.4 to 16.5 ms, the index compiled from source each process. Recompute: time the two parses in fresh processes and a `timeit` over the 4,000 names in each tree, with `PYTHONDONTWRITEBYTECODE` unset and the index's bytecode written, and the first parse again with it set and an empty cache prefix. 5. **Not done here: a listed suffix word's kind.** The index lookup this change makes does not settle 4b's `Jr` and `Sr` (`tests/v2/pipeline/test_assemble.py`'s strict expected failures stand): the kind a token takes is named by `_vocab.suffix_kind`, which also names `Ambiguity.alternative`, so reading the kind from the index at assemble alone would split the two, and reading it inside `suffix_kind` moves the alternatives and puts the index on the decision path. 4b decides it. diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py index 1199363aa..206245ab1 100644 --- a/nameparser/_pipeline/_assemble.py +++ b/nameparser/_pipeline/_assemble.py @@ -134,7 +134,7 @@ def _part_languages(text: str, kind: ReadingKind, ('Lt.Gov.', 'MD/PhD', 'MD;'), sorted: a suffix's parts divided by any mark but the period first, each an acronym ('M.D./Ph.D.'), then any word's by every mark. Read only for a word the parse's - own lexicon does not list whole (`_files`).""" + own lexicon does not list whole (`_listed_whole`).""" kinds = _kinds(kind, role) if kinds is None: return () @@ -157,23 +157,25 @@ def _by_parts(n: str, kinds: frozenset[str], keep: str) -> tuple[str, ...]: kinds)})) -def _files(text: str, kind: ReadingKind, role: Role, - lexicon: Lexicon) -> tuple[str, ...]: - """Reading.languages for a word whose whole form no file holds - (`_languages` empty): the files of the listed words it joins, save - where the parse's own lexicon lists it whole, so that the parse - read it whole and not by its parts ('zzq-dr' a caller added as a - title names no file, though 'dr' is in `mul`). Asked only after - `_languages` comes back empty, so a word a file holds pays nothing.""" - found = _part_languages(text, kind, role) - if found: - n = _normalize(text) - bare = n.replace(".", "") - for name in _VOCAB_FIELDS: - field = getattr(lexicon, name) - if n in field or bare in field: - return () - return found +@functools.lru_cache(maxsize=4096) +def _forms(text: str) -> tuple[str, str]: + """`text` as the vocabulary stores it, and without its periods.""" + n = _normalize(text) + return n, n.replace(".", "") + + +def _listed_whole(text: str, lexicon: Lexicon) -> bool: + """Whether the parse's own lexicon lists `text` whole, in either + form, so that the parse read it whole and not by its parts: a + caller's 'zzq-dr' title names none of the files of 'dr'. Asked + only of a word whose whole form no file holds and whose parts some + file does (`_part_languages`), so other words pay no frame.""" + n, bare = _forms(text) + for name in _VOCAB_FIELDS: + field = getattr(lexicon, name) + if n in field or bare in field: + return True + return False def _split_partner(tokens: tuple[WorkToken, ...], dropped: set[int], @@ -233,16 +235,21 @@ def _alternative(pending: PendingAmbiguity, materialized: tuple[Token, ...], elif isinstance(role, (Deferred, Beside)): # assign found no name position to name it from return None - if middle_as_family and role is Role.MIDDLE: - role = Role.FAMILY # the files holding the reading for any of its words, as a token's - # own reading names them + # own reading names them, by the field before the policy's fold of + # the middle (a middle word names the files listing it as given) found: set[str] = set() if alt.source in _LISTED: # a loop, not a generator, which would resume a frame per word for t in materialized: - found.update(_languages(t.text, alt.kind, role) - or _files(t.text, alt.kind, role, lexicon)) + langs = _languages(t.text, alt.kind, role) + if (not langs + and (langs := _part_languages(t.text, alt.kind, role)) + and _listed_whole(t.text, lexicon)): + langs = () + found.update(langs) + if middle_as_family and role is Role.MIDDLE: + role = Role.FAMILY return Reading(alt.kind, role, alt.source, tuple(sorted(found))) @@ -278,6 +285,7 @@ def assemble(state: ParseState) -> ParsedName: dropped = set(state.dropped) final: dict[int, Token] = {} surnames = state.lexicon.surnames + folds_middle = state.policy.middle_as_family tokens = state.tokens last = len(tokens) # the suffix reading of the token before, and whether it opens a @@ -337,11 +345,12 @@ def assemble(state: ParseState) -> ParsedName: # of the two words, which the second then takes written = (t.text + _split_partner(tokens, dropped, i) if prev_split else t.text) - reading = (_reading(kind, role, source, langs) - if source in _LISTED - and (langs := _languages(written, kind, role) - or _files(written, kind, role, - state.lexicon)) + langs = _languages(written, kind, role) if source in _LISTED else () + if (not langs and source in _LISTED + and (langs := _part_languages(written, kind, role)) + and _listed_whole(written, state.lexicon)): + langs = () + reading = (_reading(kind, role, source, langs) if langs else _READINGS[kind, role, source]) before = reading else: @@ -404,10 +413,16 @@ def assemble(state: ParseState) -> ParsedName: # a particle or connective with nothing to join reads # as the name word it stands as ('van' in 'van Gogh') kind, source = ReadingKind.NAME, name_source(tags) - reading = (_reading(kind, role, source, langs) - if source in _LISTED - and (langs := _languages(t.text, kind, role) - or _files(t.text, kind, role, state.lexicon)) + # a name's files by its field, a middle word the policy + # folded into the family by the middle it stands as + field = (Role.MIDDLE if role is Role.FAMILY and folds_middle + and FOLDED_TAG in tags else role) + langs = _languages(t.text, kind, field) if source in _LISTED else () + if (not langs and source in _LISTED + and (langs := _part_languages(t.text, kind, field)) + and _listed_whole(t.text, state.lexicon)): + langs = () + reading = (_reading(kind, role, source, langs) if langs else _READINGS[kind, role, source]) final[i] = Token(t.text, t.span, role, tags, reading) # No alphanumeric character among the SURVIVING tokens means no diff --git a/nameparser/_types.py b/nameparser/_types.py index f9ce29ced..cece4d997 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -957,9 +957,11 @@ class Reading: #: family name names ``en``, which lists it so, and not ``fil``, #: which lists it as a given name). Empty for a reading no file #: holds: a shape, - #: position alone, a word a caller's own ``Lexicon`` adds that no - #: file holds in that category, or a shipped list kept outside the - #: per-word files (the Korean surnames that divide ``김민준``). + #: position alone, a name read into a field no file lists it in + #: (``van`` as a family name, ``nl`` listing it as given), a word a + #: caller's own ``Lexicon`` adds that no file holds in that + #: category, or a shipped list kept outside the per-word files (the + #: Korean surnames that divide ``김민준``). #: A suffix reading is matched by the word's suffix entry whatever #: its finer kind, a word having one. An ``Ambiguity.alternative`` #: names the files holding it for any word the report names. Not diff --git a/tests/v2/pipeline/test_assemble.py b/tests/v2/pipeline/test_assemble.py index 19564dde6..e64223a54 100644 --- a/tests/v2/pipeline/test_assemble.py +++ b/tests/v2/pipeline/test_assemble.py @@ -445,6 +445,12 @@ def test_a_suffix_word_reads_as_its_vocabulary_kind( # own field: `ma` is family in en, given in en and fil ("Long Ma", "Ma", ("en",)), ("Ma Long", "Ma", ("en", "fil")), + ("Jane Doe (nee Smith Ma)", "Ma", ("en",)), + # a declared name in a field no file lists it in names none: fr + # lists `fils` only as a family name + ("Alves, Pedro Fils", "Fils", ()), + # a bound given name names its files in the family field too + ("John Abdul", "Abdul", ("ar",)), ("Abu Bakar Smith", "Abu", ("ar",)), # an acronym by its periods removed, and a credential split across # a space, listed whole, each of its words naming the files @@ -487,18 +493,21 @@ def test_a_reading_names_the_files_holding_it( assert tok.reading.languages == languages -@pytest.mark.parametrize("field,word,text", [ - ("titles", "zzq-dr", "Zzq-Dr John Smith"), - ("particles", "mac-van", "John mac-van Smith"), +@pytest.mark.parametrize("field,word,text,written", [ + ("titles", "zzq-dr", "Zzq-Dr John Smith", "Zzq-Dr"), + ("particles", "mac-van", "John mac-van Smith", "mac-van"), + # listed without the periods it is written with + ("titles", "drma", "Dr.Ma Smith", "Dr.Ma"), ]) def test_a_callers_whole_word_names_no_file_of_its_parts( - field: str, word: str, text: str) -> None: + field: str, word: str, text: str, written: str) -> None: # the parse read it whole, so the files of `dr` or `van` are not # its reading's; the default's own joined words still name theirs - # (`Lt.Gov.` in _EXPECTED_LANGUAGES) + # (`Lt.Gov.` in _EXPECTED_LANGUAGES, and `Dr.Ma` below) + assert parse(text).tokens[0].reading is not None lex = Lexicon.default().add(**{field: {word}}) (tok,) = [t for t in Parser(lexicon=lex).parse(text).tokens - if t.text.lower() == word] + if t.text == written] assert tok.reading is not None assert tok.reading.source is ReadingSource.VOCABULARY assert tok.reading.languages == () @@ -516,6 +525,28 @@ def test_a_reading_from_a_callers_own_vocabulary_names_no_file() -> None: assert readings["St"] is not None and readings["St"].languages == () +def test_the_default_reads_a_joined_word_by_its_parts() -> None: + (tok,) = parse("Dr.Ma Smith").tokens[:1] + assert tok.reading is not None and tok.reading.languages == ("mul",) + + +@pytest.mark.parametrize("text,word", [ + ("John Ma Smith", "Ma"), ("Smith, John Ma", "Ma")]) +def test_a_folded_middle_names_the_files_of_the_middle_it_stands_as( + text: str, word: str) -> None: + # middle_as_family reports the field, and the files stay those + # listing the word as the given name it stands as + pn = Parser(policy=Policy(middle_as_family=True)).parse(text) + (tok,) = [t for t in pn.tokens if t.text == word] + assert tok.role is Role.FAMILY and tok.reading is not None + assert tok.reading.languages == ("en", "fil") + (amb,) = Parser(policy=Policy(middle_as_family=True)).parse( + "Doe, John van DO").ambiguities + assert amb.alternative is not None + assert amb.alternative.role is Role.FAMILY + assert amb.alternative.languages == ("en", "nl") + + def test_an_alternative_names_the_files_holding_it() -> None: (amb,) = parse("JOHN SMITH MA").ambiguities assert amb.alternative is not None From a75f692f053909b636056a56b6bd9b6be756d837 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 29 Oct 2026 15:29:25 -0700 Subject: [PATCH 7/7] Cover Reading.languages' lexicon check; fold two unreachable branches (#754) Codecov flagged five lines of the diff. The two that clear a reading's files when the parse's own Lexicon lists the word whole are reachable, for a token and for a report's alternative, and a test now reaches each (a caller's 'MD-ZZ' and 'MA-ZZ'); each fails under its own mutation. The other three no input reaches: a kind no file row holds is read only by shape, so _kinds answers an empty set instead of None, and a split credential's first word always has its second after it, since classify tags them as a pair, so the partner lookup ends in an expression with an empty default rather than a branch. Co-Authored-By: Claude Opus 5.5 --- nameparser/_pipeline/_assemble.py | 24 +++++++++++------------- tests/v2/pipeline/test_assemble.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py index 206245ab1..b5ee09568 100644 --- a/nameparser/_pipeline/_assemble.py +++ b/nameparser/_pipeline/_assemble.py @@ -101,9 +101,10 @@ def _kinds_of(*categories: str) -> frozenset[str]: _LISTED = frozenset({ReadingSource.VOCABULARY, ReadingSource.DECLARED}) -def _kinds(kind: ReadingKind, role: Role) -> frozenset[str] | None: +def _kinds(kind: ReadingKind, role: Role) -> frozenset[str]: + # an initial or a patronymic matches no row return (_NAME_KINDS.get(role, _ANY_NAME_KINDS) - if kind is ReadingKind.NAME else _KINDS.get(kind)) + if kind is ReadingKind.NAME else _KINDS.get(kind, frozenset())) # Both lookups are memos, C-level on a hit, so a word read before costs @@ -118,8 +119,6 @@ def _languages(text: str, kind: ReadingKind, role: Role) -> tuple[str, ...]: matched by ('M.A' in 'John Smith (M.A)') and a word listed whole without them ('Lt.Gen.' as 'ltgen').""" kinds = _kinds(kind, role) - if kinds is None: - return () n = _normalize(text) found = languages_of(n, kinds) if not found and "." in n: @@ -136,8 +135,6 @@ def _part_languages(text: str, kind: ReadingKind, then any word's by every mark. Read only for a word the parse's own lexicon does not list whole (`_listed_whole`).""" kinds = _kinds(kind, role) - if kinds is None: - return () n = _normalize(text) found = _by_parts(n, kinds, ".") if kinds is _SUFFIX_KINDS else () return found or _by_parts(n, kinds, "") @@ -185,13 +182,14 @@ def _split_partner(tokens: tuple[WorkToken, ...], dropped: set[int], between them ('Ph. "Bob" D.'), or nothing where that word is not the joined suffix word that completes it. Asked only of a split credential's first word, so other names pay nothing.""" - for j in range(i + 1, len(tokens)): - if j in dropped or tokens[j].role is Role.NICKNAME: - continue - nxt = tokens[j] - return (nxt.text if nxt.role is Role.SUFFIX and "joined" in nxt.tags - else "") - return "" + # classify tags a split's words only as a pair, so its second word + # always follows; an empty default keeps parse total without a + # branch no input reaches + nxt = next((tokens[j] for j in range(i + 1, len(tokens)) + if j not in dropped and tokens[j].role is not Role.NICKNAME), + None) + return (nxt.text if nxt is not None and nxt.role is Role.SUFFIX + and "joined" in nxt.tags else "") _reading = functools.lru_cache(maxsize=1024)(Reading) diff --git a/tests/v2/pipeline/test_assemble.py b/tests/v2/pipeline/test_assemble.py index e64223a54..739b72342 100644 --- a/tests/v2/pipeline/test_assemble.py +++ b/tests/v2/pipeline/test_assemble.py @@ -525,6 +525,23 @@ def test_a_reading_from_a_callers_own_vocabulary_names_no_file() -> None: assert readings["St"] is not None and readings["St"].languages == () +def test_a_callers_whole_suffix_and_its_alternative_name_no_file_of_its_parts( + ) -> None: + # 'md' and 'ma' are in en, and the default reads 'MD-ZZ' by its + # parts; listed whole, the token and the report's alternative name + # none of their files + lex = Lexicon.default().add(suffix_acronyms={"md-zz", "ma-zz"}, + suffix_acronyms_ambiguous={"ma-zz"}) + parser = Parser(lexicon=lex) + (md,) = [t for t in parser.parse("John Smith MD-ZZ").tokens + if t.text == "MD-ZZ"] + assert md.reading == Reading("credential", "suffix", "vocabulary") # type: ignore[arg-type] + assert md.reading is not None and md.reading.languages == () + (amb,) = parser.parse("JOHN SMITH MA-ZZ").ambiguities + assert amb.alternative == Reading("name", "family", "declared") # type: ignore[arg-type] + assert amb.alternative is not None and amb.alternative.languages == () + + def test_the_default_reads_a_joined_word_by_its_parts() -> None: (tok,) = parse("Dr.Ma Smith").tokens[:1] assert tok.reading is not None and tok.reading.languages == ("mul",)