From 2b420f063a4774969de09a7a8e9305557ff73c67 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 31 Oct 2026 17:46:01 -0700 Subject: [PATCH 1/7] Build file-strict: parser_for(..., strict=True) compiles only the chosen files (#753, 3j) A strict choice compiles the chosen languages' vocabulary files, each tag selecting its own and its ancestors', plus native-script vocabulary, and nothing else; mul.toml and und.toml leave. It applies as a delta, as a plain choice does. A strict call choosing no language raises with a fix that runs; a strict choice accepts a file holding no reading (vi), which a plain one still refuses. _reading_index.strict_ungated derives the words strict ungates against the plain choice of the same tags, held to the reviewed set (16 words, 27 word-and-tag pairs). The index now lists every file's tag (FILES). The CLI gains --strict and falls back from a pack code to a language tag. Co-Authored-By: Claude Opus 5.5 --- nameparser/__main__.py | 43 ++++++++++++- nameparser/_parser.py | 72 +++++++++++++++++++--- nameparser/_reading_index.py | 95 +++++++++++++++++++++++++---- nameparser/config/_index.py | 91 +++++++++++++++++++++++++++- tests/v2/test_cli.py | 43 +++++++++++++ tests/v2/test_parser.py | 107 ++++++++++++++++++++++++++++++++- tests/v2/test_reading_index.py | 78 +++++++++++++++++++++++- tools/vocab/build.py | 18 ++++-- 8 files changed, 513 insertions(+), 34 deletions(-) diff --git a/nameparser/__main__.py b/nameparser/__main__.py index 106d975ca..9389ae426 100644 --- a/nameparser/__main__.py +++ b/nameparser/__main__.py @@ -4,6 +4,8 @@ python -m nameparser --json "Doe, John" python -m nameparser --order family-first "Kovács János Péter" python -m nameparser --locale ja "山田太郎" + python -m nameparser --locale ko --strict "Kim Do Hyun" + python -m nameparser --locale vi --strict "Do Van Minh" """ import argparse import dataclasses @@ -52,14 +54,25 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("--locale", metavar="CODE", help="parse with a locale pack (e.g. 'ru'); see " "nameparser.locales. 'ja' divides unspaced names " - "when nameparser[ja] is installed") + "when nameparser[ja] is installed. A code no " + "pack has but a vocabulary file does (e.g. 'ko') " + "chooses that language alone") + ap.add_argument("--strict", action="store_true", + help="read only the --locale language's vocabulary " + "file and native-script words, dropping every " + "other language's (for single-origin names); a " + "pack choosing no language chooses its code") ap.add_argument("--order", choices=_ORDERS, help="the order to read a name's words in " "(default: given-first); a Han, Hangul or kana " "name keeps its script's order, and a family " "comma fixes the order") args = ap.parse_args(argv) + if args.strict and args.locale is None: + ap.error("--strict needs --locale: it keeps only the chosen " + "language's vocabulary") pack = None + unknown = None segmenter: Segmenter | None = None noted = False # `is not None`, not truthiness: --locale "" must reach get() and @@ -69,7 +82,24 @@ def main(argv: list[str] | None = None) -> int: try: pack = locales.get(args.locale) except KeyError as exc: - ap.error(exc.args[0]) # exits 2, message to stderr + # not a pack: a language tag chooses that language alone, + # which parser_for checks below; if it is neither, the error + # lists the codes and the tags + unknown = exc.args[0] + try: + pack = Locale(code="cli_language", lexicon=Lexicon.empty(), + languages=frozenset({args.locale})) + except (TypeError, ValueError): + ap.error(unknown) # exits 2, message to stderr + # a strict call needs a language: a pack choosing none (zh, ja, + # ru) chooses its own code, which parser_for checks below + if args.strict and not pack.languages: + try: + pack = dataclasses.replace( + pack, languages=frozenset({pack.code})) + except ValueError: + ap.error(f"--strict: the pack {pack.code!r} chooses no " + f"language, and its code is not one") # The pack only activates division; the installed extra divides. # Without it a note says so in the command line's terms, in # place of the parser's warning (rules/D1), whose other remedies @@ -96,7 +126,14 @@ def main(argv: list[str] | None = None) -> int: if noted: warnings.filterwarnings( "ignore", message="Policy.segment_scripts activates") - parser = parser_for(*packs, segmenter=segmenter) + try: + parser = parser_for(*packs, segmenter=segmenter, + strict=args.strict) + except ValueError as exc: + # the choice's own refusal, without the pack wrapper naming + # the CLI's stand-in code + why = str(exc.__cause__ or exc).replace("strict=True", "--strict") + ap.error(f"{unknown}; nor a language: {why}" if unknown else why) n = parser.parse(args.name) if args.json: print(json.dumps(n.as_dict(), ensure_ascii=False)) diff --git a/nameparser/_parser.py b/nameparser/_parser.py index 41c7a8195..01e98ff04 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -286,25 +286,29 @@ def parse(text: str) -> ParsedName: return _default_parser().parse(text) -def _reading_index_selected(languages: frozenset[str]) -> frozenset[str]: +def _reading_index_selected(languages: frozenset[str], + strict: bool = False) -> frozenset[str]: # the reading index loads on first lookup, so a caller who chooses # no language never imports it (decisions/vocabulary-model, 3a) from nameparser._reading_index import selected - return selected(languages) + return selected(languages, strict=strict) @functools.cache -def _choice_delta(chosen: frozenset[str]) -> tuple[ +def _choice_delta(chosen: frozenset[str], strict: bool = False) -> tuple[ dict[str, tuple[frozenset[str], frozenset[str]]], tuple[frozenset[str], tuple[tuple[str, str], ...]]]: """What the choice of the file tags `chosen` changes against the default's compile: per Lexicon field, the entries that leave and the ones that join, and the case masks that leave and the ones set. A delta, so it applies to a caller's own Lexicon as well as to the - default (the RFC's "Where the provenance lives at runtime").""" + default (the RFC's "Where the provenance lives at runtime"); a + `strict` choice's delta is the same comparison, its compile holding + only the chosen files and native-script vocabulary.""" from nameparser._lexicon import _vocab_fields from nameparser._reading_index import compile_choice - default, choice = compile_choice(frozenset()), compile_choice(chosen) + default = compile_choice(frozenset()) + choice = compile_choice(chosen, strict) was, now = _vocab_fields(default), _vocab_fields(choice) fields = {name: (was[name] - now[name], now[name] - was[name]) for name in was if was[name] != now[name]} @@ -315,14 +319,15 @@ def _choice_delta(chosen: frozenset[str]) -> tuple[ return fields, masks -def _apply_choice(lexicon: Lexicon, languages: frozenset[str]) -> Lexicon: +def _apply_choice(lexicon: Lexicon, languages: frozenset[str], + strict: bool = False) -> Lexicon: """`lexicon` under the choice of `languages`, a delta against the default's compile: what the choice drops leaves `lexicon` whoever put it there, a caller included, what it turns on joins, and every other entry stays as the caller had it.""" from nameparser._lexicon import _SUBSET_FIELDS fields, (unmasked, masks) = _choice_delta( - _reading_index_selected(languages)) + _reading_index_selected(languages, strict), strict) updates: dict[str, object] = { name: (getattr(lexicon, name) - gone) | joined for name, (gone, joined) in fields.items()} @@ -344,7 +349,8 @@ def _apply_choice(lexicon: Lexicon, languages: frozenset[str]) -> Lexicon: def parser_for(*locales: Locale, base: Parser | None = None, - segmenter: Segmenter | None | _Unset = UNSET) -> Parser: + segmenter: Segmenter | None | _Unset = UNSET, + strict: bool = False) -> Parser: """The packs' ``languages`` unioned into one choice, applied to base's lexicon first (so the order of the packs does not change it); then lexicon fragments unioned left-to-right onto the result; @@ -359,6 +365,23 @@ def parser_for(*locales: Locale, base: Parser | None = None, ``do``, where ``parser_for(PT, EN)`` chooses both and keeps both (decisions/vocabulary-model, 2026-10-29, chained choices). + ``strict=True`` makes the choice strict: only the chosen languages' + vocabulary files are compiled, each tag selecting its own and its + ancestors' as a plain choice does, plus native-script vocabulary (a + word with a letter and no Latin letter); every other file's + readings leave, mul.toml's and und.toml's among them, and with + them the markers they derive. It is for single-origin data + (``parser_for(Locale(code="ko", lexicon=Lexicon.empty(), + languages=frozenset({"ko"})), strict=True)`` reads ``Kim Do Hyun`` + with no Portuguese particle ``do``). It applies as a delta, as a + plain choice does, so a caller's own listing of a dropped word + leaves too; the packs' fragments and policy patches apply after it + unchanged. A strict choice accepts a language whose file holds no + reading yet (``vi``), which a plain one refuses as changing + nothing, and raises ValueError when the packs choose no language, + since it would then strip every Latin-script reading + (decisions/vocabulary-model, 2026-10-30 #753, file-strict). + Validation errors raised while applying a pack are wrapped with that pack's identity (rule D2) -- PolicyPatch validates lazily, so with stacked packs the raw error @@ -379,6 +402,8 @@ def parser_for(*locales: Locale, base: Parser | None = None, lexicon and policy by hand.""" if base is not None and not isinstance(base, Parser): raise TypeError(f"base must be a Parser or None, got {base!r}") + if not isinstance(strict, bool): + raise TypeError(f"strict must be True or False, got {strict!r}") for loc in locales: if not isinstance(loc, Locale): raise TypeError(f"parser_for() takes Locale packs, got {loc!r}") @@ -400,14 +425,16 @@ def parser_for(*locales: Locale, base: Parser | None = None, for loc in locales: if loc.languages: try: - _reading_index_selected(loc.languages) + _reading_index_selected(loc.languages, strict) except ValueError as exc: raise ValueError( f"while applying locale {loc.code!r}: {exc}") from exc languages |= loc.languages + if strict and not languages: + raise ValueError(_no_strict_language(locales)) if languages: try: - lexicon = _apply_choice(lexicon, languages) + lexicon = _apply_choice(lexicon, languages, strict) except (TypeError, ValueError) as exc: # rules/D2: "applying a locale pack wraps any such error with # the locale's code, so a stacked configuration names which @@ -467,3 +494,28 @@ def parser_for(*locales: Locale, base: Parser | None = None, for w in caught: warnings.warn(w.message, stacklevel=2) return built + + +def _no_strict_language(locales: tuple[Locale, ...]) -> str: + """The refusal of a strict call that chooses no language, which + would keep only native-script vocabulary and strip every Latin one + in silence. The fix it offers is a pack choosing the first pack's + code where that is a language a strict choice accepts, else + ``en``.""" + tag = "en" + for loc in locales[:1]: + try: + _reading_index_selected(frozenset({loc.code}), True) + except ValueError: + pass + else: + tag = loc.code + codes = ", ".join(repr(loc.code) for loc in locales) + named = ("no pack was given" if not locales + else f"the pack {codes} chooses none" if len(locales) == 1 + else f"the packs {codes} choose none") + return (f"strict=True keeps only the chosen languages' vocabulary, and " + f"{named}, so it would strip every Latin-script reading; add " + f"a pack that chooses a language, e.g. " + f"Locale(code={tag!r}, lexicon=Lexicon.empty(), " + f"languages=frozenset({{{tag!r}}})), or drop strict=True") diff --git a/nameparser/_reading_index.py b/nameparser/_reading_index.py index ce1351422..7e71c5530 100644 --- a/nameparser/_reading_index.py +++ b/nameparser/_reading_index.py @@ -29,6 +29,7 @@ from __future__ import annotations import functools +import unicodedata from collections.abc import Iterable, Mapping from dataclasses import dataclass from types import MappingProxyType @@ -243,7 +244,9 @@ def readings_of(text: str) -> tuple[IndexReading, ...]: #: The tags no choice names: mul.toml's readings are every language's #: and und.toml's no known language's, so neither is a language to -#: choose, and a choice never removes a reading of either (choose). +#: choose. A plain choice never removes a reading of either (choose); a +#: strict one, which keeps only the chosen files, drops both +#: (:func:`strict_pool`). UNCHOOSABLE = frozenset({"mul", "und"}) @@ -253,28 +256,49 @@ def tags() -> frozenset[str]: return frozenset(row[0] for rows in _by_text().values() for row in rows) -def selected(languages: Iterable[str]) -> frozenset[str]: +@functools.cache +def files() -> frozenset[str]: + """The tags of every shipped vocabulary file, one holding no + reading (only exclusions) included: :func:`tags` and the files a + strict choice may name, which a plain one may not.""" + from nameparser.config._index import FILES + return frozenset(FILES) + + +def selected(languages: Iterable[str], *, + strict: bool = False) -> frozenset[str]: """The file tags a choice of `languages` selects: each chosen tag's own and its ancestors', never its descendants' (``pt-BR`` selects ``pt-BR`` and ``pt``; ``pt`` only ``pt``), matched without regard to case, as BCP 47 tags are. No macrolanguage table is read: ``ms`` does not select ``id``. + A plain choice selects among the files that hold a reading, since a + file holding none would change nothing; a `strict` one among every + file, since it drops whatever is not chosen and so is never inert + (``vi``, whose file holds only exclusions). + Raises ValueError for ``mul`` or ``und``, and for a tag that selects no file at all, naming the tags that do.""" - by_fold = {t.casefold(): t for t in tags() - UNCHOOSABLE} + held = (files() if strict else tags()) - UNCHOOSABLE + by_fold = {t.casefold(): t for t in held} out: set[str] = set() for tag in languages: if tag.casefold() in UNCHOOSABLE: raise ValueError( f"{tag!r} is not a language to choose: mul.toml's readings " - f"are every language's and und.toml's no known language's, " - f"and no choice removes a reading of either; drop it") - parts = tag.split("-") - found = {by_fold[a] for a in ("-".join(parts[:i]).casefold() - for i in range(len(parts), 0, -1)) - if a in by_fold} + f"are every language's and und.toml's no known language's; " + f"a plain choice never removes a reading of either, and a " + f"strict one drops both. Drop it") + found = {by_fold[a] for a in _ancestors(tag) if a in by_fold} if not found: + empty = sorted(t for t in files() - tags() - UNCHOOSABLE + if t.casefold() in _ancestors(tag)) + if empty: + raise ValueError( + f"{empty[0]}.toml holds no reading yet, so a plain " + f"choice of {tag!r} would change nothing; a strict one " + f"(strict=True) accepts it") raise ValueError( f"no vocabulary file holds the language {tag!r} or one it " f"belongs to; the languages that have one: " @@ -283,6 +307,12 @@ def selected(languages: Iterable[str]) -> frozenset[str]: return frozenset(out) +def _ancestors(tag: str) -> list[str]: + """`tag` and the tags it belongs to, casefolded, longest first.""" + parts = tag.split("-") + return ["-".join(parts[:i]).casefold() for i in range(len(parts), 0, -1)] + + def choose(readings: Iterable[IndexReading], chosen: frozenset[str]) -> list[IndexReading]: """The readings a choice of the file tags `chosen` compiles, each @@ -324,13 +354,54 @@ def choose(readings: Iterable[IndexReading], return out +def is_native_script(text: str) -> bool: + """Whether a strict choice keeps `text` whatever its file: it holds + a letter and no Latin letter (the CJK honorifics, the Cyrillic and + Devanagari titles), so it selects itself by script, which is read + from the word as the parse always reads it. A word with no letter + (``&``, ``2``) is not native-script.""" + letters = [c for c in text if c.isalpha()] + return bool(letters) and not any( + "LATIN" in unicodedata.name(c, "") for c in letters) + + +def strict_pool(chosen: frozenset[str]) -> list[IndexReading]: + """The readings a strict choice of the file tags `chosen` compiles + from, before :func:`choose`: the chosen files' and native-script + vocabulary's, and nothing else, mul.toml's and und.toml's included + (file-strict; decisions/vocabulary-model, 2026-10-30).""" + return [r for r in index() if r.tag in chosen or is_native_script(r.text)] + + @functools.cache -def compile_choice(chosen: frozenset[str]) -> Compiled: +def compile_choice(chosen: frozenset[str], strict: bool = False) -> Compiled: """The constants the shipped index compiles to under a choice of the file tags `chosen` (:func:`selected`'s result), kept per choice: a compile costs milliseconds, paid once per choice and only by a - caller who makes one. An empty choice compiles the default.""" - return compile_readings(choose(index(), chosen)) + caller who makes one. An empty plain choice compiles the default; a + `strict` one compiles from :func:`strict_pool` alone.""" + pool = strict_pool(chosen) if strict else index() + return compile_readings(choose(pool, chosen)) + + +def strict_ungated(chosen: frozenset[str]) -> frozenset[str]: + """The words a strict choice of the file tags `chosen` ungates: a + chosen file holds the word's vocabulary reading and only a dropped + one the name reading gating it, so under strict a possibly-given + particle is never-given, a title only inside a phrase is a title, + and a suffix gated by the count is not. Compared with the PLAIN + choice of the same tags, never with the default's markers, which + miss a reading the choice turns on (decisions/vocabulary-model, + 2026-10-30 #753, file-strict).""" + plain, strict = compile_choice(chosen), compile_choice(chosen, True) + out = ((plain["PARTICLES"] - plain["NON_GIVEN_NAME_PARTICLES"]) + & strict["NON_GIVEN_NAME_PARTICLES"]) + out |= plain["TITLES_AMBIGUOUS"] & strict["TITLES"] + out |= plain["SUFFIX_ACRONYMS_AMBIGUOUS"] & ( + strict["SUFFIX_ACRONYMS"] - strict["SUFFIX_ACRONYMS_AMBIGUOUS"]) + out |= plain["SUFFIX_WORDS_AMBIGUOUS"] & ( + strict["SUFFIX_WORDS"] - strict["SUFFIX_WORDS_AMBIGUOUS"]) + return frozenset(out) def changed_words(chosen: frozenset[str]) -> frozenset[str]: diff --git a/nameparser/config/_index.py b/nameparser/config/_index.py index 4867cf676..f63c573ae 100644 --- a/nameparser/config/_index.py +++ b/nameparser/config/_index.py @@ -6,7 +6,8 @@ One line per word in the vocabulary files, holding a row per reading, on by default or not: (tag, kind, properties), the tag being the file's BCP 47 tag and the properties the ones the compiler reads, as sorted -(key, value) pairs. +(key, value) pairs; then FILES, every file's tag, one holding no +reading included. Read it through nameparser._reading_index (index, readings_of), which documents the row; _compiled.py is this index compiled. """ @@ -1877,3 +1878,91 @@ ('양', (('ko', 'honorific', ()),)), ('주니어', (('ko', 'generation', ()),)), ) + +FILES: tuple[str, ...] = ( + 'ak', + 'am', + 'ar', + 'as', + 'be', + 'bg', + 'bin', + 'bn', + 'bo', + 'bs', + 'ca', + 'cs', + 'da', + 'de', + 'el', + 'en', + 'es', + 'et', + 'fa', + 'ff', + 'fi', + 'fil', + 'fr', + 'ga', + 'gd', + 'gu', + 'ha', + 'he', + 'hi', + 'hr', + 'hu', + 'hy', + 'id', + 'is', + 'it', + 'ja', + 'ka', + 'kk', + 'km', + 'kn', + 'ko', + 'kok', + 'lo', + 'lt', + 'lv', + 'mi', + 'mk', + 'ml', + 'mn', + 'mr', + 'ms', + 'mul', + 'my', + 'nb', + 'ne', + 'nl', + 'nn', + 'or', + 'pa', + 'pl', + 'pt', + 'pt-BR', + 'ro', + 'ru', + 'sd', + 'shn', + 'si', + 'sk', + 'sl', + 'sr', + 'sv', + 'sw', + 'ta', + 'te', + 'th', + 'ti', + 'tr', + 'uk', + 'und', + 'ur', + 'vi', + 'yi', + 'yo', + 'zh', + 'zu', +) diff --git a/tests/v2/test_cli.py b/tests/v2/test_cli.py index 3fe1a8e7a..afa776b01 100644 --- a/tests/v2/test_cli.py +++ b/tests/v2/test_cli.py @@ -257,3 +257,46 @@ def factory() -> Segmenter: _installed(monkeypatch, factory) _fields(capsys, "--locale", "zh", "王小明") assert calls == [] + + +# --strict (#753, 3j) and --locale's fallback to a language tag + +@pytest.mark.parametrize(("argv", "roles"), [ + (("--locale", "ko", "--strict", "Kim Do Hyun"), ("Kim", "Do", "Hyun")), + (("--locale", "vi", "--strict", "Do Van Minh"), ("Do", "Van", "Minh")), + # a pack choosing no language chooses its code under --strict + (("--locale", "zh", "--strict", "De Xu"), ("De", "", "Xu")), + # without --strict, ko's file holds nothing Latin to prefer + (("--locale", "ko", "Kim Do Hyun"), ("Kim", "", "Do Hyun")), +]) +def test_cli_locale_falls_back_to_a_language_and_strict_reads_it_alone( + capsys: pytest.CaptureFixture[str], argv: tuple[str, ...], + roles: tuple[str, str, str]) -> None: + assert _roles(_fields(capsys, *argv)) == roles + + +def test_cli_strict_without_a_locale_is_a_usage_error() -> None: + proc = _run("Kim Do Hyun", "--strict") + assert proc.returncode == 2 + assert "--strict needs --locale" in proc.stderr + + +def test_cli_locale_neither_a_pack_nor_a_language_lists_both() -> None: + proc = _run("John Smith", "--locale", "xx", "--strict") + assert proc.returncode == 2 + # the pack codes, then the language tags + assert "tr_az" in proc.stderr and "nor a language" in proc.stderr + assert "ko" in proc.stderr.split("nor a language")[1] + + +def test_cli_a_language_with_no_reading_needs_strict() -> None: + proc = _run("Do Van Minh", "--locale", "vi") + assert proc.returncode == 2 + assert "vi.toml holds no reading yet" in proc.stderr + assert "(--strict) accepts it" in proc.stderr + + +def test_cli_strict_with_a_pack_whose_code_is_no_language() -> None: + proc = _run("John Smith", "--locale", "tr_az", "--strict") + assert proc.returncode == 2 + assert "chooses no language, and its code is not one" in proc.stderr diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 3896923e8..9726b342c 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -2511,7 +2511,8 @@ def test_an_error_applying_a_choice_names_the_choice( import nameparser._parser as parser_mod orphan: object = ({"particles_ambiguous": (frozenset(), frozenset({"zzq"}))}, (frozenset(), ())) - monkeypatch.setattr(parser_mod, "_choice_delta", lambda chosen: orphan) + monkeypatch.setattr(parser_mod, "_choice_delta", + lambda chosen, strict=False: orphan) with pytest.raises(ValueError, match="while applying the languages chosen, en, pt: " "particles_ambiguous marks"): @@ -2529,7 +2530,7 @@ def test_a_choice_moves_case_masks_as_it_moves_words( real = ri.compile_choice default = real(frozenset()) - def stand_in(chosen: frozenset[str]) -> object: + def stand_in(chosen: frozenset[str], strict: bool = False) -> object: if not chosen: return default masks = {k: v for k, v in default["CAPITALIZATION_EXCEPTIONS"].items() @@ -2538,7 +2539,8 @@ def stand_in(chosen: frozenset[str]) -> object: monkeypatch.setattr(ri, "compile_choice", stand_in) delta = parser_mod._choice_delta.__wrapped__(frozenset({"pt"})) # type: ignore[attr-defined] assert delta == ({}, (frozenset({"phd"}), (("zzq", "ZzQ"),))) - monkeypatch.setattr(parser_mod, "_choice_delta", lambda chosen: delta) + monkeypatch.setattr(parser_mod, "_choice_delta", + lambda chosen, strict=False: delta) base = dataclasses.replace(Lexicon.default(), capitalization_exceptions=( *Lexicon.default().capitalization_exceptions, ("abcq", "AbcQ"))) got = parser_mod._apply_choice(base, frozenset({"pt"})) @@ -2597,3 +2599,102 @@ def test_a_choice_that_changes_nothing_leaves_the_lexicon_itself() -> None: assert changed_words(selected(["sv"])) == frozenset() mine = Lexicon.default().add(titles={"zzq"}) assert parser_mod._apply_choice(mine, frozenset({"sv"})) is mine + + +# -- strict (#753, 3j: file-strict, decisions/vocabulary-model's +# 2026-10-30 #753, file-strict entry) --------------------------------- + +@pytest.mark.parametrize("tags,name,before,after", [ + # romanized Korean: every word below is a common Korean syllable + # another file holds a vocabulary reading of + (("ko",), "Kim Do Hyun", {"family": "Do Hyun"}, # pt's particle + {"middle": "Do", "family": "Hyun"}), + (("ko",), "Kim Bin Hyun", {"family": "Bin Hyun"}, # ar's and ms's + {"middle": "Bin", "family": "Hyun"}), + (("ko",), "Ra Min Jun", {"title": "Ra"}, # de's academic title + {"given": "Ra", "title": ""}), + (("ko",), "Kim Min Ra", {"suffix": "Ra"}, # en's credential + {"family": "Ra", "suffix": ""}), + # what adding en costs: its credential is back + (("ko", "en"), "Kim Min Ra", {"suffix": "Ra"}, {"suffix": "Ra"}), + (("ja",), "Gen Hoshino", {"title": "Gen"}, {"given": "Gen"}), + (("zh",), "De Xu", {"family": "De Xu"}, + {"given": "De", "family": "Xu"}), + # vi.toml holds no reading, which only a strict choice accepts + (("vi",), "Do Van Minh", {"family": "Van Minh"}, + {"middle": "Van", "family": "Minh"}), +]) +def test_a_strict_choice_reads_only_the_chosen_files( + tags: tuple[str, ...], name: str, before: dict[str, str], + after: dict[str, str]) -> None: + default = parse(name) + for field, value in before.items(): + assert getattr(default, field) == value, field + got = parser_for(_choosing(*tags), strict=True).parse(name) + for field, value in after.items(): + assert getattr(got, field) == value, field + + +def test_a_strict_choice_drops_mul_and_keeps_native_script_vocabulary() -> None: + strict = parser_for(_choosing("vi"), strict=True) + # mul.toml's `dr` leaves; the period still makes a title by shape + assert strict.parse("Dr Minh Do").given == "Dr" + assert strict.parse("Dr. Minh Do").title == "Dr." + # a native-script word selects itself by script, whatever its file + assert strict.parse("김민준 씨").suffix == "씨" + # and the Korean surname list is outside the per-word model + assert strict.lexicon.surnames == Lexicon.default().surnames + + +@pytest.mark.parametrize("packs,match", [ + ((), "no pack was given"), + (("ZH",), "the pack 'zh' chooses none"), + (("ZH", "RU"), "the packs 'zh', 'ru' choose none"), +]) +def test_a_strict_call_choosing_no_language_is_refused_with_a_fix_that_runs( + packs: tuple[str, ...], match: str) -> None: + from nameparser import locales + given = tuple(getattr(locales, p) for p in packs) + with pytest.raises(ValueError, match=match) as info: + parser_for(*given, strict=True) + # the message hands the reader a pack to add: it must run + offered = re.search(r"e\.g\. (Locale\(.*\)\)), or", str(info.value)) + assert offered is not None + pack = eval(offered.group(1), {"Locale": Locale, "Lexicon": Lexicon}) + assert parser_for(*given, pack, strict=True).lexicon != Lexicon.default() + + +def test_a_file_with_no_reading_is_a_strict_choice_only() -> None: + with pytest.raises(ValueError, match=r"vi\.toml holds no reading yet.*" + r"strict=True"): + parser_for(_choosing("vi")) + assert "do" not in parser_for(_choosing("vi"), strict=True).lexicon.particles + + +def test_a_strict_choice_still_refuses_mul_and_an_unknown_tag() -> None: + for tag, match in (("mul", "a strict one drops both"), + ("xx", "no vocabulary file holds")): + with pytest.raises(ValueError, match=match): + parser_for(_choosing(tag), strict=True) + + +def test_strict_is_a_bool() -> None: + with pytest.raises(TypeError, match="strict must be True or False"): + parser_for(_choosing("ko"), strict=1) # type: ignore[arg-type] + + +def test_a_strict_choice_is_a_delta_over_the_union_in_any_order() -> None: + # a caller's listing of a dropped word leaves, a word no file holds + # stays (measured in the decisions entry); the union, then strict + base = Parser(lexicon=Lexicon.default().add(titles={"dr", "zzq"})) + got = parser_for(_choosing("vi"), base=base, strict=True).lexicon + assert "zzq" in got.titles and "dr" not in got.titles + ko, en = _choosing("ko", code="aa"), _choosing("en", code="bb") + assert (parser_for(ko, en, strict=True).lexicon + == parser_for(en, ko, strict=True).lexicon + == parser_for(_choosing("ko", "en"), strict=True).lexicon) + + +def test_a_packs_fragment_is_unioned_on_after_the_strict_compile() -> None: + pack = _choosing("ko", lexicon=Lexicon.empty().add(suffix_acronyms={"ra"})) + assert parser_for(pack, strict=True).parse("Kim Min Ra").suffix == "Ra" diff --git a/tests/v2/test_reading_index.py b/tests/v2/test_reading_index.py index e71bfab47..629489386 100644 --- a/tests/v2/test_reading_index.py +++ b/tests/v2/test_reading_index.py @@ -113,7 +113,7 @@ def test_every_property_the_compiler_reads_survives_the_index() -> None: def test_the_committed_index_is_what_the_toml_writes() -> None: pool = build.load(build.read_dir()) - fresh = build.render_index(build.index_rows(pool)) + fresh = build.render_index(build.index_rows(pool), build.file_tags(pool)) assert build.index_output().read_text(encoding="utf-8") == fresh, ( f"nameparser/config/_index.py is stale against nameparser/vocab/; " f"run: {build.REFRESH}") @@ -294,3 +294,79 @@ def test_prefer_never_drops_a_reading_of_unknown_language() -> None: def test_holds_kind_reads_every_file_whatever_the_default( text: str, kind: str, held: bool) -> None: assert ri.holds_kind(text, kind) is held + + +# -- strict (#753, 3j) ------------------------------------------------ + +@pytest.mark.parametrize("text,native", [ + ("씨", True), ("господин", True), ("श्री", True), ("殿", True), + ("dr", False), ("ông", False), ("&", False), ("2", False), +]) +def test_native_script_is_a_letter_and_no_latin_letter( + text: str, native: bool) -> None: + assert ri.is_native_script(text) is native + + +def test_a_strict_pool_holds_the_chosen_files_and_native_script_only() -> None: + chosen = ri.selected(["pt-BR"], strict=True) + pool = ri.strict_pool(chosen) + assert {r.tag for r in pool if not ri.is_native_script(r.text)} == chosen + # mul.toml and und.toml are another language's file under strict + assert not [r for r in pool if r.tag in ri.UNCHOOSABLE + and not ri.is_native_script(r.text)] + native = [r for r in ri.index() if ri.is_native_script(r.text)] + assert native and all(r in pool for r in native) + + +def test_a_file_holding_no_reading_is_chosen_only_strictly() -> None: + empty = ri.files() - ri.tags() + assert empty and ri.tags() <= ri.files() + for tag in sorted(empty): + with pytest.raises(ValueError, match=f"{tag}.toml holds no reading"): + ri.selected([tag]) + assert ri.selected([tag], strict=True) == {tag} + # nothing Latin is left to compile: the native-script pool alone + assert ri.compile_choice(frozenset({tag}), True) == ri.compile_readings( + ri.choose([r for r in ri.index() if ri.is_native_script(r.text)], + frozenset({tag}))) + + +#: The words each strict choice of one tag ungates against the plain +#: choice of the same tag (decisions/vocabulary-model's 2026-10-30 #753, +#: file-strict entry: 16 words, 27 word-and-tag pairs over 13 tags). A +#: word leaves this table when a name reading is written in the chosen +#: file whose names carry it, and joins it when a chosen file's +#: vocabulary reading loses its gate: review each move. +STRICT_UNGATED = { + "ar": {"cheikh", "sultan"}, + "cs": {"st"}, + "fr": {"la", "marquis", "marquise", "st"}, + "ga": {"mac"}, + "gd": {"mac", "nic"}, + "hr": {"st"}, + "it": {"del", "la", "san", "santa"}, + "ms": {"bin", "sultan"}, + "nl": {"den", "tho"}, + "pt": {"da", "do", "santa"}, + "pt-BR": {"da", "do", "santa"}, + "sk": {"st"}, + "sl": {"st"}, +} + + +def test_the_words_strict_ungates_are_the_reviewed_set() -> None: + got = {} + for tag in sorted(ri.files() - ri.UNCHOOSABLE): + words = ri.strict_ungated(ri.selected([tag], strict=True)) + if words: + got[tag] = set(words) + assert got == STRICT_UNGATED + + +def test_strict_ungated_compares_with_the_plain_choice_not_the_default() -> None: + # cs turns on the generation `st`, gated by en's given reading; the + # default's markers hold no generation `st` at all, so a comparison + # with them would miss it + cs = frozenset({"cs"}) + assert "st" not in ri.compile_choice(frozenset())["SUFFIX_WORDS"] + assert "st" in ri.strict_ungated(cs) diff --git a/tools/vocab/build.py b/tools/vocab/build.py index 9d2fd8667..4b3572c88 100644 --- a/tools/vocab/build.py +++ b/tools/vocab/build.py @@ -1389,7 +1389,8 @@ def render(compiled: Compiled) -> str: One line per word in the vocabulary files, holding a row per reading, on by default or not: (tag, kind, properties), the tag being the file's BCP 47 tag and the properties the ones the compiler reads, as sorted -(key, value) pairs. +(key, value) pairs; then FILES, every file's tag, one holding no +reading included. Read it through nameparser._reading_index (index, readings_of), which documents the row; _compiled.py is this index compiled. """ @@ -1435,11 +1436,13 @@ def index_output() -> Path: return OUTPUT.with_name("_index.py") -def render_index(rows: Iterable[Any]) -> str: +def render_index(rows: Iterable[Any], files: Iterable[str]) -> str: """One word per line, its rows in index order, so a change to a word's readings is a one-line diff; the whole tuple is one constant, which a cached import loads without building anything, and grouped - by word, so a lookup table is one dict() call.""" + by word, so a lookup table is one dict() call. FILES follows: the + tag of every file, one holding only exclusions included, which a + strict choice may name (_reading_index.files).""" out = [INDEX_HEADER, "", "READINGS: tuple[tuple[str, tuple[tuple[str, str, " "tuple[tuple[str, object], ...]], ...]], ...] = ("] @@ -1447,10 +1450,17 @@ def render_index(rows: Iterable[Any]) -> str: for r in rows: words.setdefault(r.text, []).append(_ri.to_row(r)) out += [f" ({text!r}, {tuple(rs)!r})," for text, rs in words.items()] + out += [")", "", "FILES: tuple[str, ...] = ("] + out += [f" {tag!r}," for tag in sorted(files)] out.append(")") return "\n".join(out) + "\n" +def file_tags(pool: Pool) -> list[str]: + """The tag of every file the pool was loaded from.""" + return sorted(name.removesuffix(".toml") for name in pool.entries) + + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("--check", action="store_true", @@ -1477,7 +1487,7 @@ def _run(*, check: bool) -> int: compiled = compile_pool(pool) stale = 0 for path, text in ((OUTPUT, render(compiled)), - (index_output(), render_index(index_rows(pool))), + (index_output(), render_index(index_rows(pool), file_tags(pool))), (rules_output(), render_rules(pool.rules))): stale += _refresh(path, text, check=check) return 1 if stale else 0 From 5ec5f665781affe487aae6885c4ad53de4b7bbfb Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 31 Oct 2026 17:49:53 -0700 Subject: [PATCH 2/7] Docs: file-strict built and its API decided (#753, 3j) The RFC's strict section, its status and 3j's row, decisions/vocabulary-model's 2026-10-31 phase 3j entry (the four API points, the CLI, the ORCID table re-run on the built code matching every cell), its Standing and Open lines, mechanisms.md#LOCALE-PACKS-PURE-DATA and AGENTS.md. docs/locales.rst gains "Reading one language only" with romanized Korean as the worked example, docs/usage.rst the --strict flag, and the release log a bullet. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- docs/design/decisions/vocabulary-model.md | 17 ++++- docs/design/mechanisms.md | 2 +- docs/design/vocabulary-model-rfc.md | 26 ++++---- docs/locales.rst | 78 ++++++++++++++++++++++- docs/release_log.rst | 1 + docs/usage.rst | 12 +++- 7 files changed, 118 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 342dab5c1..39410e071 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -206,7 +206,7 @@ Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely o ### Vocabulary (`nameparser/vocab/`) and the configuration layer (`nameparser/config/`) -**The default vocabulary is written per word, in TOML, and compiled.** `nameparser/vocab/.toml` holds one file per BCP 47 language tag, each `[[word]]` entry a `text` and its readings keyed by kind. What a file means, how an entry is built, the kinds and properties with the set each compiles into, exclusions, bearer counts and the steps for adding a word are in `nameparser/vocab/README.md`, the reference for anyone editing the TOML; `tests/v2/test_vocab.py` holds its kind, property, derived-marker and reserved-name tables to the names `build.py` accepts and the sets it compiles, so read it rather than restating it here. The parser never reads the TOML: `tools/vocab/build.py` runs the build checks that replaced the modules' import-time asserts (its docstring numbers them), turns every reading into an index row with its file's tag, and writes the rows as the committed reading index `nameparser/config/_index.py` (#753, phase 3a) and, compiled, as the committed `nameparser/config/_compiled.py`, frozenset literals one entry per line; the build never reads `_index.py` back. A file a shipped pack reads may also hold a `[rules]` table, each key a `Policy` field (`patronymic_rules`, `name_order`, `two_family_names`, `two_given_names`), which the build checks against its `RULE_KEYS` and `PACK_RULES` and writes as `nameparser/config/_rules.py`; `_locale.rules_patch(tag)` turns a file's table into the `PolicyPatch` a pack carries, RU's and TR_AZ's from `ru.toml` and `tr.toml`, and the `two_family_names` of ES, CS and SK from `es.toml`, `cs.toml` and `sk.toml` (#754, 4f; decisions/vocabulary-model's 2026-10-29 4f entry). The compiler is `nameparser/_reading_index.py`'s `compile_readings`, with the kind and property tables it reads (`CATEGORY`, `BEHAVIORAL`, `GATED_WORDS`), so the build and a choice of languages compile alike: `Locale.languages` (3.0, phase 3b) is a choice of BCP 47 tags, which `parser_for` unions across its packs and applies as a delta against the default's compile, `choose` there selecting the readings (a chosen file's `default = false` ones on; on a word a chosen file holds a vocabulary reading of, other files' vocabulary readings off, save `mul.toml`'s and `und.toml`'s; a chosen conjunction's `may_be_initial` dropped), and `changed_words` deriving a choice's deviation surface, which tests/v2/test_locales.py's gate reads (decisions/vocabulary-model, 2026-10-29 3b); the build keeps the checks and loads that module by path, as it loads `_lexicon.py`, and `tests/v2/test_reading_index.py` holds that the index compiled with no choice is `_compiled.py` exactly. `readings_of(text)` there is the run-time reader of the index, every reading of a word with its file's tag, loaded on first call; `languages_of(text, categories)` is the reader a parse calls, from assemble, for `Reading.languages` (3.0), so the first parse that reads a word from its `Lexicon`'s vocabulary loads the index. Each `config/` module re-exports its constants from `_compiled.py` (`TITLES = _compiled.TITLES`) under its autodoc docstring, so the public names and the docs did not move. `tests/v2/test_vocab.py` fails when `_compiled.py` is stale, and `tests/v2/test_reading_index.py` when the index is, each naming the refresh command, and `test_vocab.py` gives every build check a mutation. The model, the phasing and what was declined: decisions/vocabulary-model. +**The default vocabulary is written per word, in TOML, and compiled.** `nameparser/vocab/.toml` holds one file per BCP 47 language tag, each `[[word]]` entry a `text` and its readings keyed by kind. What a file means, how an entry is built, the kinds and properties with the set each compiles into, exclusions, bearer counts and the steps for adding a word are in `nameparser/vocab/README.md`, the reference for anyone editing the TOML; `tests/v2/test_vocab.py` holds its kind, property, derived-marker and reserved-name tables to the names `build.py` accepts and the sets it compiles, so read it rather than restating it here. The parser never reads the TOML: `tools/vocab/build.py` runs the build checks that replaced the modules' import-time asserts (its docstring numbers them), turns every reading into an index row with its file's tag, and writes the rows, and every file's tag (`FILES`, a file holding only exclusions included, for a strict choice to name), as the committed reading index `nameparser/config/_index.py` (#753, phase 3a) and, compiled, as the committed `nameparser/config/_compiled.py`, frozenset literals one entry per line; the build never reads `_index.py` back. A file a shipped pack reads may also hold a `[rules]` table, each key a `Policy` field (`patronymic_rules`, `name_order`, `two_family_names`, `two_given_names`), which the build checks against its `RULE_KEYS` and `PACK_RULES` and writes as `nameparser/config/_rules.py`; `_locale.rules_patch(tag)` turns a file's table into the `PolicyPatch` a pack carries, RU's and TR_AZ's from `ru.toml` and `tr.toml`, and the `two_family_names` of ES, CS and SK from `es.toml`, `cs.toml` and `sk.toml` (#754, 4f; decisions/vocabulary-model's 2026-10-29 4f entry). The compiler is `nameparser/_reading_index.py`'s `compile_readings`, with the kind and property tables it reads (`CATEGORY`, `BEHAVIORAL`, `GATED_WORDS`), so the build and a choice of languages compile alike: `Locale.languages` (3.0, phase 3b) is a choice of BCP 47 tags, which `parser_for` unions across its packs and applies as a delta against the default's compile, `choose` there selecting the readings (a chosen file's `default = false` ones on; on a word a chosen file holds a vocabulary reading of, other files' vocabulary readings off, save `mul.toml`'s and `und.toml`'s; a chosen conjunction's `may_be_initial` dropped; under `parser_for(..., strict=True)`, 3j, only the chosen files and native-script vocabulary compiled, through `strict_pool`), `changed_words` deriving a choice's deviation surface and `strict_ungated` the words a strict choice ungates against the plain one, which tests/v2/test_locales.py's gate reads (decisions/vocabulary-model, 2026-10-29 3b); the build keeps the checks and loads that module by path, as it loads `_lexicon.py`, and `tests/v2/test_reading_index.py` holds that the index compiled with no choice is `_compiled.py` exactly. `readings_of(text)` there is the run-time reader of the index, every reading of a word with its file's tag, loaded on first call; `languages_of(text, categories)` is the reader a parse calls, from assemble, for `Reading.languages` (3.0), so the first parse that reads a word from its `Lexicon`'s vocabulary loads the index. Each `config/` module re-exports its constants from `_compiled.py` (`TITLES = _compiled.TITLES`) under its autodoc docstring, so the public names and the docs did not move. `tests/v2/test_vocab.py` fails when `_compiled.py` is stale, and `tests/v2/test_reading_index.py` when the index is, each naming the refresh command, and `test_vocab.py` gives every build check a mutation. The model, the phasing and what was declined: decisions/vocabulary-model. The `config/` SETS are frozen since 2.2 (#293): there is no `.add()`/`.remove()` on any of them, so a default word list is changed at runtime by configuring a `Lexicon`, never by editing the constant, and the shipped default is changed by editing the TOML and rebuilding. **Neither dict was frozen, and `CAPITALIZATION_EXCEPTIONS` is not covered by anything else either.** `REGEXES` is a compiled-pattern table rather than vocabulary and was never in #293's scope; `CAPITALIZATION_EXCEPTIONS` is vocabulary-shaped and is a decided, in-scope exemption. So the hazard the freeze closes is still live for it: `CAPITALIZATION_EXCEPTIONS['dclinpsy'] = 'DClinPsy'` after the first parse misses the cached `Lexicon.default()`. Same advice — configure the object (`dataclasses.replace(lexicon, capitalization_exceptions=...)`). `tests/v2/test_contracts.py::test_every_vocabulary_constant_is_frozen` names both dicts as explicit exemptions rather than letting its `isinstance` filter drop them. diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index 0e4844e2b..6157edaee 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -13,7 +13,8 @@ The vocabulary was a set of base sets plus marker subsets (`particles` and `part - A shipped pack that is only a choice of languages declares no `DEVIATES`, its surface the words its choice changes; `ga` turns on `nic`, `mhic`, `mic` and `uí` as clan particles, `gd` `nic`, and `pt-BR` `sb` as Sobrinho's spelling, while the single letters `f` and `n` are excluded, the pack included, the letter abbreviating a surname more often than the agnomen in the filings that write it — 2026-10-30 #753, phase 3h. - `es` turns on the courtesy, academic and professional titles that lead a Spanish name with no complement (`lic`, `licenciado`, `licenciada`, `lcdo`, `lcda`, `ingeniero`, `ingeniera`, `arq`, `mtro`, `mtra`, `doctora`, `profesor`, `profesora`, `doña`) and `pt` `doutor`, `doutora`, `professora` and `profa`, `pt-BR` with it; `don`, `dona` and `dom` are excluded, the packs included, names opening with each as a name more often than as the title; `maestro`, a family name whose form of address is written `Mtro.`, is excluded too, and so is `h`, the hijo an initial is written as; `sr` keeps a generation reading beside its salutation in both files, so a choice leaves it the dual position decides; no connective continues a title run, the shape unwritten in name data, and the office titles that take a complement stay out — 2026-10-30 #753, phase 3g. - Of the romanized titles a pack's phase lists, one whose native spelling is a default title turns on unless, counted in persons and pooled across every population measured, names open with it more often than it opens one as the title; a spelling outside the phase's list (`Shreemati`, `Srimati`) stays open until a phase writes it as a reading: `hi` turns on `smt`, `shrimati` and `shriyut`, `bn` `smt`, `shrimati` and `adhyapak` (`shriyut` with no bearer anywhere, `adhyapak` with family-name bearers only, every one where the title claim, acting at the head, does not reach), while `shri`, opening given names in every population measured, stays excluded, and `sushri`, an Odia given name, waits for strict with no reading written, a plain choice being made for data that holds those names — 2026-10-30 #753, phase 3e. -- A language pack is a choice of languages on `Locale.languages` selecting which readings compile, a tag choosing its own file and its ancestors', and a reading index, loaded on its first lookup, carries the languages to run time; strict is file-strict, the chosen files and native-script vocabulary only, `mul.toml` and `und.toml` not among them, so it needs no 3i, and its API beyond the `strict=True` argument is not yet decided; third-party language files are deferred — 2026-10-27 #753 (Derek); 2026-10-29 #753, phase 3a (Derek); 2026-10-29 #753, phase 3b; 2026-10-30 #753, file-strict. +- A language pack is a choice of languages on `Locale.languages` selecting which readings compile, a tag choosing its own file and its ancestors', and a reading index, loaded on its first lookup, carries the languages to run time; strict is file-strict, the chosen files and native-script vocabulary only, `mul.toml` and `und.toml` not among them, so it needs no 3i; third-party language files are deferred — 2026-10-27 #753 (Derek); 2026-10-29 #753, phase 3a (Derek); 2026-10-29 #753, phase 3b; 2026-10-30 #753, file-strict. +- A strict choice is `parser_for(..., strict=True)` over the whole choice, applied as a delta as a plain choice is; a strict call whose packs choose no language raises; a strict choice accepts a tag whose file holds no reading, which a plain one refuses as changing nothing; a bare choice has no constructor, no `locales.choose`; and the words strict ungates are derived against the plain choice of the same tags and held to a reviewed set by a test — 2026-10-31 #753, phase 3j. - A choice is matched without regard to case, a tag whose file does not exist selects its ancestors' (`pt-PT` selects `pt`), `mul` and `und` are no language to choose, and a reading turned on also prefers, so `de`'s generation `i` drops `ca`'s and `pl`'s conjunction under a `de` choice; a choice is one call's: one chained through `base=` applies on top of the base's rather than unioning with it, and a choice drops the caller's own base entries it removes — 2026-10-29 #753, phase 3b; 2026-10-29 #753, phase 3b, chained choices (Derek). - A reading right in its own language and wrong only as a default, because another language written in the same script collides with it, is written `default = false` in its file, never excluded, so that after the packs an exclusion means wrong everywhere: cs `st` since phase 2h, and cs `ml`, hr, sk and sl `ml` and `st` and fr `père` since 3c, the rest in their packs' PRs — 2026-10-27 #753 (Derek); 2026-10-29 #753, phase 3c. - A name reading written `default = false` derives no marker in the default and counts for no check guarding the default's compile; a particle whose word's only `given` readings are off is refused until a pack needs one — 2026-10-29 #753, phase 3a. @@ -54,7 +55,7 @@ The vocabulary was a set of base sets plus marker subsets (`particles` and `part Not standing: the plans the classifications carried out, phase 1's mechanical conversion of the unclassified categories into `und.toml` (2026-10-18 #737) and the shared readings left in place until their categories were classified (2026-10-19 #749), both done by 2026-10-20 #751, the conjunctions; and #747's reason for exempting `und.toml`, its entries keeping the Python lists' order for its vetting comments to point along (2026-10-19 #747), overtaken once it held two entries and no such comment. -Open: #913 (the title words phase 2j left, read as titles though borne as names), #753 (phase 3, language packs; file-strict's API beyond `strict=True`, a strict call choosing no language and whether bare choices get a constructor, and which mul.toml words move so that strict `en` can shed them; whether a `Parser` records its choice, so that chained choices union, `Parser.languages`, decided with the configure-by-reading method names), #754 (phase 4, order, shapes and the output model). +Open: #913 (the title words phase 2j left, read as titles though borne as names), #753 (phase 3, language packs; which mul.toml words move so that strict `en` can shed them, and the 16 words strict ungates, each read for a name reading in the chosen file; whether a `Parser` records its choice, so that chained choices union, `Parser.languages`, decided with the configure-by-reading method names), #754 (phase 4, order, shapes and the output model). ## Log @@ -413,7 +414,7 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) - **Recompute:** `uv run pytest -n0 tests/v2/test_shape_inventory.py`; the counts by order from `ARRANGEMENTS` and `SHAPES` (each row's order is `SHAPES[row.shape].order`); the corpus and snapshot counts from `git diff --numstat` against `1f6bdf1d` after `uv run python tools/differential/build_shapes_corpus.py` and `uv run python tools/differential/snapshot.py --record`; the control by parsing each family-first row with `Parser()`. - **Disclosed, not added:** the two title-particles at the head written with periods (`Freiherr St. van Berg MA`, `Freiherr. St. van Berg MA`) read as the bare spelling does, the title run whole, under every order. - **Open, unweighed and held by no issue, so these rows carry no mark:** an ambiguous particle opening a family-first name is left where position puts it and reported, as root AGENTS.md's particles paragraph describes, though no decision weighs it there (`van Gogh Vincent` under `FAMILY_FIRST`: family `van`, given `Gogh`; `du Ménil Thi Mai`, its letter decomposed, under `FAMILY_FIRST_GIVEN_LAST`: family `du`, middle `Ménil Thi`); rules/P1 covers a never-given particle only. And `Vega y Lopez Jr., PhD Jones` under the family-first orders reads given `y`, middle `Lopez` (`FAMILY_FIRST`) and given `Lopez`, middle `y` (`FAMILY_FIRST_GIVEN_LAST`): the default order's division of the words laid out by the order, which decisions/C3 records under the default order only. And a Hungarian title after a married `-né` word, before the woman's birth family name (`Bakainé dr. Korbács Melinda`), 102 of 37,250 names in the 2024 municipal candidates and every one misread today (decisions/H5's #938 entry has the measurement): what licenses the title there is the `-né` word, so it is a question for a future `hu` pack, which no sub-phase schedules, and not for a general rule. -- **2026-10-30 #753, file-strict — STRICT IS FILE-STRICT: A STRICT CHOICE READS ONLY THE CHOSEN LANGUAGES' FILES AND NATIVE-SCRIPT VOCABULARY, NOT `mul.toml` OR `und.toml`, SO IT NEEDS NO 3i; THE `strict=True` ARGUMENT STANDS, THE REST OF ITS API IS PROPOSED, NOT DECIDED, AND NOTHING IS BUILT.** SUPERSEDES: decision 6's strict variant in the 2026-10-15/16 entry, and in the 2026-10-27 #753 entry item 2's definition of strict (other languages' name readings and their markers leave), item 4 (strict waits until name readings say whose names they are) and 3i in item 11. Item 1's `strict=True` argument of `parser_for`, over the whole choice, stands. The definition and items 1 to 3 were decided in chat on 2026-10-30; the measurements were taken on 2026-10-31 on master `b577e9a8`. The design overview is docs/design/vocabulary-model-rfc.md's "Strict: the chosen files and nothing else". +- **2026-10-30 #753, file-strict (its proposed `locales.choose` and its bullet that strict `vi` needs vi.toml readings first superseded by 2026-10-31 #753, phase 3j) — STRICT IS FILE-STRICT: A STRICT CHOICE READS ONLY THE CHOSEN LANGUAGES' FILES AND NATIVE-SCRIPT VOCABULARY, NOT `mul.toml` OR `und.toml`, SO IT NEEDS NO 3i; THE `strict=True` ARGUMENT STANDS, THE REST OF ITS API IS PROPOSED, NOT DECIDED, AND NOTHING IS BUILT.** SUPERSEDES: decision 6's strict variant in the 2026-10-15/16 entry, and in the 2026-10-27 #753 entry item 2's definition of strict (other languages' name readings and their markers leave), item 4 (strict waits until name readings say whose names they are) and 3i in item 11. Item 1's `strict=True` argument of `parser_for`, over the whole choice, stands. The definition and items 1 to 3 were decided in chat on 2026-10-30; the measurements were taken on 2026-10-31 on master `b577e9a8`. The design overview is docs/design/vocabulary-model-rfc.md's "Strict: the chosen files and nothing else". 1. **The definition.** A strict choice compiles only the chosen languages' files, each tag selecting its own and its ancestors', plus native-script vocabulary, which stays on because it selects itself by script (the CJK honorifics; the Korean surname list, outside the per-word model, is in no choice's compile and stays too); everything else leaves, vocabulary and name readings alike. A word with no reading in what is compiled is read by position and shape alone, so under strict `vi`, `do`, which only en.toml and pt.toml hold, is a name word, while the shape rules still act (`Dr. Minh Do` keeps title `Dr.`, rules/H2; `Minh Do, M.D., Ph.D.` keeps suffix `Ph.D.` past the second comma, rules/C2, but reads given `M.D.`). For a credential, the harm item 4 of the 2026-10-27 entry guarded against (`Minh Do` → suffix `Do`) does not arise today, measured 2026-10-31: no strict choice of one tag ungates a credential acronym, each credential's gating name reading sitting in its own file. It holds only while name readings are added to more files and never moved: moving `do`'s Đỗ readings from en.toml to vi.toml would ungate its credential under strict `en`. For the other kinds it can, where the vocabulary reading is in a chosen file and the gating name reading only in a dropped one (the ungated-words bullet below; strict `pt` reads `Do Van Minh` family `Do Van Minh`). 3i, the attribute naming each name reading's bearers' languages, is still not needed for strict: under file-strict that case is remedied by data, a name reading written in the chosen file whose names carry the word. Name-strict, the earlier definition, under which other languages' titles, credentials and particles stayed and only their name readings left, is set aside with 3i. 2. **`mul.toml` and `und.toml` are not in a strict choice.** `mul.toml` stays as it is: its entries name several languages each or "the European languages" (`dr`, `prof`), or carry no note (the numerals, `&`), so under file-strict it behaves as one more language's file. Open: entries may later move between en.toml and mul.toml, or into the files of the languages that write them, so that strict `en` can shed problem words; which words those are is not yet known. 3. **Dropping English credentials under strict `vi` is accepted in principle.** Vietnamese writes its degrees in Vietnamese, before the name (`TS.`, `ThS.`, `GS.`, `PGS.`, `BS.`, stacked as `PGS.TS.`), and has no generational suffix; English `PhD` and `MD` come with English-context writing. Measured below rather than assumed. @@ -446,3 +447,13 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) - **3i's evidence, kept** from the RFC paragraph this entry retires, should 3i be wanted again: all measured 2026-10-28 on master `c8357e74`, not on this entry's tree: of the 227 name readings, 165 words carried one; `do`'s family box led by Brazil in ORCID (none of Brazil's 7,227 `do` alone, 2,629 opening with it and 4,598 holding it inside) but by share Vietnam (1.90% against 0.81%; PatentsView 1.78% and 0.64%), Vietnam having 618 boxes of `do` alone and Korea 151; `kim`'s family box 17.67% of ORCID's Korean holders and 13.30% of PatentsView's, its given box 0.93% of PatentsView's Danish inventors. Recompute: `uv run python tools/name_usage.py do kim --no-wikidata --boxes`, and the whole-box split from the cached ORCID extract's `family` and `country` keys. - **Candidate next steps, not taken here:** Vietnamese degree prefixes as vi.toml readings, each checked for collisions and bearers per nameparser/vocab/AGENTS.md first; `ông` and `bà` stay excluded, Ông being a Vietnamese surname, a clash within Vietnamese strict does not settle; 3h's `nl` prepositions and 3e's `sushri` as data with the clashing name readings declared in ko.toml, vi.toml and or.toml, gated under the plain choice and not under strict, though the plain `nl` choice still reads `Park In Hwan` family `In Hwan` and `Nguyen To Nga` family `To Nga`, item 4's recorded cost, the marker acting at the head only (re-run 2026-10-31), which a follow-up must settle before shipping them; `_reading_index`'s `UNCHOOSABLE` comment and `selected`'s refusal text, which say no choice removes a reading of `mul` or `und`, updated with 3j; the 16 ungated words read one by one. - **Recompute:** a strict choice of the tags `T` is `compile_readings(choose(rows, T))` from `nameparser._reading_index`, `rows` the `index()` rows whose tag is in `T` or whose text has a letter and no letter whose `unicodedata.name` contains `LATIN`, applied to `Lexicon.default()` as `_parser._apply_choice` applies `_choice_delta` (the delta taken against `compile_choice(frozenset())`, the markers of a dropped base entry removed), and a pack's own patch and fragment applied by `parser_for(dataclasses.replace(pack, languages=frozenset()), base=Parser(lexicon=that))`. Count a word's share of the moves as the moves whose name, split at spaces and commas with edge periods dropped and casefolded, holds it. Score ORCID with `local_agreement.read(record, local_agreement.FIELDED["orcid-researchers"], parser, local_agreement._title_words(parser.lexicon))` over the cache's `raw.jsonl` records of each country, and PSC the same with `"psc-individuals"` and the `nationality` key; the title-word filter is the given box's first word, split at spaces and periods and casefolded, in {dr, prof, professor, mr, mrs, ms, miss, dra, ing, mgr, assoc, assist, er, engr, drs, ir}. The ungated words: for each choosable tag, the words in a marker (`particles_ambiguous`, `suffix_acronyms_ambiguous`, `suffix_words_ambiguous`, `titles_ambiguous`) of the plain choice of that tag (`parser_for(Locale(code="x", lexicon=Lexicon.empty(), languages={tag})).lexicon`) that stay in the base set of the strict lexicon and leave the marker, or join `titles`; compared with `Lexicon.default()`'s markers instead, it misses the readings a choice turns on (`st` under cs, hr, sk and sl, `nic` under gd) and finds 15 words. The scripts are in no tree: they are a few dozen lines each from this recipe. +- **2026-10-31 #753, phase 3j — FILE-STRICT IS BUILT: `parser_for(..., strict=True)` COMPILES ONLY THE CHOSEN FILES AND NATIVE-SCRIPT VOCABULARY; A STRICT CALL CHOOSING NO LANGUAGE RAISES; NO `locales.choose`; THE WORDS STRICT UNGATES ARE DERIVED AND HELD BY A TEST; A FILE HOLDING NO READING IS A STRICT CHOICE ONLY; THE DEFAULT DOES NOT MOVE.** SUPERSEDES: in the 2026-10-30 #753, file-strict entry, the proposed `locales.choose(*tags)` and the files-with-no-readings bullet's "strict `vi` needs vi.toml readings first". Carries out 3j of item 11 of the 2026-10-27 #753 entry, with the definition of the 2026-10-30 entry. The four API points below were decided on 2026-10-31; the rest of the 2026-10-30 entry's proposed bullet is built as proposed (no `Locale.strict` field; the chosen files' `default = false` readings turned on and no other file's; patches and fragments applied after the strict compile, untouched; `Lexicon.given_names` and `Lexicon.surnames` outside it; packs stacked as a union; a caller's listing of a dropped word dropped by the delta). Built on master `8438ccbb`. + 1. **A strict call that chooses no language raises `ValueError`.** The message names the packs given and offers a pack that chooses a language, the first pack's code where a strict choice accepts it as one (`parser_for(locales.ZH, strict=True)` offers `Locale(code='zh', lexicon=Lexicon.empty(), languages=frozenset({'zh'}))`), else `en`; tests/v2/test_parser.py runs the offered code. + 2. **No public `locales.choose`.** A bare choice is a `Locale` with an empty lexicon and the tags, the spelling docs/usage.rst teaches. + 3. **The words strict ungates are derived**, `_reading_index.strict_ungated`, each strict choice against the plain choice of the same tags and never against the default's markers, which miss a reading the choice turns on; tests/v2/test_reading_index.py holds every one-tag strict choice to a per-tag table, which is the 2026-10-30 entry's 16 words and 27 word-and-tag pairs over 13 tags exactly, so a word that joins or leaves is a reviewed diff. + 4. **A strict choice accepts a tag whose file holds no reading** (hu, kk, kok, vi and zu today): under strict it is not inert, every Latin-script reading leaving. A plain choice of one is still refused, its message now saying a strict one accepts it. The reading index records every file's tag for this (`FILES` in nameparser/config/_index.py, read by `_reading_index.files`), a file holding only exclusions writing no row of `READINGS`. + - `_reading_index`'s `UNCHOOSABLE` comment and `selected`'s refusal text, which said no choice removes a reading of `mul` or `und`, now say a plain choice never does and a strict one drops both. + - **The command line** gains `--strict`, which needs `--locale` (exit 2 without it). `--locale` falls back from a pack code to a bare choice of a language tag (`--locale ko`), and a code that is neither exits 2 listing the pack codes and the tags. Under `--strict`, a pack that chooses no language chooses its own code (`--locale zh --strict`), and one whose code is no language (`tr_az`) exits 2. These are the debug helper's, chosen with the build rather than decided as API. + - **Measured: the 2026-10-30 entry's ORCID table re-run on the built code**, its recipe with `parser_for(pack, strict=True)` (and a bare `en` pack beside it for strict + en) in place of the simulation, on this phase's branch over `8438ccbb`, Python 3.11: every cell matches, 14 countries by four columns. + - **Measured: the worked examples on the built code**, default order, against the default: under strict `ko`, `Kim Do Hyun` reads middle `Do`, family `Hyun` (default family `Do Hyun`, pt's particle), `Kim Bin Hyun` middle `Bin` (ar's and ms's particle), `Ra Min Jun` given `Ra` (de's academic title), `Kim Min Ra` family `Ra` (en's credential), back to suffix `Ra` under strict `ko` + `en`; strict `ja` reads `Gen Hoshino` given `Gen` (default title), strict `zh` `De Xu` given `De`, family `Xu` (default family `De Xu`). Every example the 2026-10-30 entry gives parses on the built code as it says. + - The default does not move: tests/v2/test_snapshot.py passes unchanged, a plain choice compiling as before. diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index c17f4dcf0..2dacc38e1 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -334,7 +334,7 @@ a fork for all ambiguous particles on "Dr. Van Jr.". **Contract statement.** A locale pack is pure data — a choice of languages, a Lexicon fragment of ADDITIONS and a Policy patch — applied by parser_for, a shipped pack's patch compiled from its language file's `[rules]` table where one is written, each key the `Policy` field of its name: the packs' languages union into one choice, which selects among the default vocabulary's readings by the file each sits in, the fragments union onto the result, and the patches fold in order; packs contain no code paths of their own. A vocabulary pack's deviations are derived from the reading index, the words whose compiled readings its choice changes; a policy or segmentation pack declares its own in its docstring and `DEVIATES`. -**How it works.** What is policy for every language (an order constant) is policy, not pack data; packs are lowercase modules exposing uppercase constants; a pack error is wrapped with the pack's code (rule D2). The fragment never removes a base entry; a choice may, and only by selection, never by a list of removals: a chosen language's `default = false` readings join; on a word a chosen language holds a vocabulary reading of, other languages' vocabulary readings leave, save `mul.toml`'s; and a chosen language's conjunction loses `may_be_initial`, its clash with the initial being a shape reading a preference outranks, which is how a `pt` choice drops `e` from `conjunctions_ambiguous` (decisions/vocabulary-model, 2026-10-27 #753, items 1, 2 and 7; built 2026-10-29, phase 3b). So a bad default is routed around only by choosing the language whose reading is right, which is why #342-class fixes still change shipped vocabulary. The choice is applied as a delta against the default's compile: it adds to the base lexicon what the choice turns on and removes from it what the choice drops, whichever of the default and the caller put a removed entry there, and leaves every other entry, a caller's own included; a pack choosing no language leaves the base's exactly as it was. The choice is one call's: a `Parser` records none, so `parser_for(EN, base=parser_for(PT))` applies `en`'s after `pt`'s and keeps neither language's `do`, where `parser_for(PT, EN)` keeps both (decisions/vocabulary-model, 2026-10-29, chained choices). Pure data means a pack can be audited: a vocabulary pack's whole content is its tag, but what it changes reaches other files too, prefer dropping their readings of its words (a `de` choice drops `ca.toml`'s conjunction `i`), so the audit is the derived surface, `changed_words`, not the pack's file alone. +**How it works.** What is policy for every language (an order constant) is policy, not pack data; packs are lowercase modules exposing uppercase constants; a pack error is wrapped with the pack's code (rule D2). The fragment never removes a base entry; a choice may, and only by selection, never by a list of removals: a chosen language's `default = false` readings join; on a word a chosen language holds a vocabulary reading of, other languages' vocabulary readings leave, save `mul.toml`'s; and a chosen language's conjunction loses `may_be_initial`, its clash with the initial being a shape reading a preference outranks, which is how a `pt` choice drops `e` from `conjunctions_ambiguous` (decisions/vocabulary-model, 2026-10-27 #753, items 1, 2 and 7; built 2026-10-29, phase 3b). A strict choice, `parser_for(..., strict=True)`, selects by file alone: only the chosen files and native-script vocabulary compile, `mul.toml`'s and `und.toml`'s readings leaving with every other file's, and it too is applied as a delta (decisions/vocabulary-model, 2026-10-30 #753, file-strict; built 2026-10-31, phase 3j). So a bad default is routed around only by choosing the language whose reading is right, which is why #342-class fixes still change shipped vocabulary. The choice is applied as a delta against the default's compile: it adds to the base lexicon what the choice turns on and removes from it what the choice drops, whichever of the default and the caller put a removed entry there, and leaves every other entry, a caller's own included; a pack choosing no language leaves the base's exactly as it was. The choice is one call's: a `Parser` records none, so `parser_for(EN, base=parser_for(PT))` applies `en`'s after `pt`'s and keeps neither language's `do`, where `parser_for(PT, EN)` keeps both (decisions/vocabulary-model, 2026-10-29, chained choices). Pure data means a pack can be audited: a vocabulary pack's whole content is its tag, but what it changes reaches other files too, prefer dropping their readings of its words (a `de` choice drops `ca.toml`'s conjunction `i`), so the audit is the derived surface, `changed_words`, not the pack's file alone. **Lives in.** nameparser/locales/ (packs), nameparser/_parser.py (parser_for, the one applier), nameparser/_reading_index.py (the selection, `choose`, and the derived surface, `changed_words`), nameparser/_locale.py (`rules_patch`, a file's compiled `[rules]` as a patch) and tools/vocab/build.py (`RULE_KEYS`, the rules a file may turn on, and `PACK_RULES`, the files the shipped packs read; decisions/vocabulary-model, 2026-10-29 4f). A rule a file turns on is a `Policy` field; a new rule kind is a core change, never code in a pack: a new `Policy` field (with `ADDED_IN`), a row in build.py's `RULE_KEYS` and in `_locale._RULE_VALUES`, and a mechanisms.md entry of its own for the rule kind. diff --git a/docs/design/vocabulary-model-rfc.md b/docs/design/vocabulary-model-rfc.md index 8608cda13..3a925bf7f 100644 --- a/docs/design/vocabulary-model-rfc.md +++ b/docs/design/vocabulary-model-rfc.md @@ -26,7 +26,7 @@ Four phases, in a fixed order: |---|---|---|---| | 1 | The per-word TOML format, compiled to today's sets | no | shipped in 2.6 (#737, #744); #749–#751 gave the words in `und.toml` their languages, leaving two particles of unrecorded origin | | 2 | Ambiguity derived from clashing readings; the ambiguity gate extended to every reading kind | yes, the first phase that does | #752, design agreed 2026-10-23; 2a, the two name-clash markers derived, landed without moving a parse (decisions/vocabulary-model, 2026-10-25) | -| 3 | Language packs that prefer or restrict their language | yes, for callers who choose one | #753, for 3.0, needs 1 and 2; design agreed 2026-10-28, 3g's connective declined 2026-10-30; 3a, the reading index and the shared compiler, 3b, the choice on `Locale.languages`, and 3c, the pack-shaped exclusions as readings off by default, landed without moving a default parse (decisions/vocabulary-model, 2026-10-29); strict redefined on 2026-10-30 as file-strict, the chosen files and native-script vocabulary only, which needs no 3i, the rest of its API proposed and the definition measured on 2026-10-31 (decisions/vocabulary-model's 2026-10-30 #753, file-strict entry) | +| 3 | Language packs that prefer or restrict their language | yes, for callers who choose one | #753, for 3.0, needs 1 and 2; design agreed 2026-10-28, 3g's connective declined 2026-10-30; 3a, the reading index and the shared compiler, 3b, the choice on `Locale.languages`, and 3c, the pack-shaped exclusions as readings off by default, landed without moving a default parse (decisions/vocabulary-model, 2026-10-29); strict redefined on 2026-10-30 as file-strict, the chosen files and native-script vocabulary only, which needs no 3i, the definition measured on 2026-10-31 (decisions/vocabulary-model's 2026-10-30 #753, file-strict entry); 3j, file-strict, landed 2026-10-31 as `parser_for(..., strict=True)`, its API decided that day (decisions/vocabulary-model's 2026-10-31 #753, phase 3j entry) | | 4 | Order and shape; the rules a pack can choose; the output model | yes | #754, 3.0 work; its output model agreed 2026-10-28, ahead of or alongside phase 3's mechanism; 4a to 4d, the reading and the views `generation`, `credentials`, `patronymic` and `surname2`, landed without moving a parse, 4e, the render defaults and `ParsedName.order`, moved the default `str()`, and 4f, `[rules]` compiled into a pack's policy patch, moved no parse (decisions/vocabulary-model, decisions/R6 and decisions/R8 to R10, 2026-10-29) | Two commitments hold through all four. **The default loads every language and keeps working for names of any language, as today**; choosing a language narrows it (decision 5 in decisions/vocabulary-model). And **languages own vocabulary and defaults, the core owns structure**: a pack configures rules the core implements, and never ships code (mechanisms.md#LOCALE-PACKS-PURE-DATA). @@ -164,7 +164,7 @@ Each sub-phase is one PR, measured on its own base: the committed snapshot read ## Phase 3: packs that prefer or restrict -Status: **agreed** (proposed 2026-10-27, #753; Derek accepted every item on 2026-10-28, six and then the other six); **3a landed 2026-10-29**, the reading index, the compiler moved into the package and `default = false` on name readings (decisions/vocabulary-model's 2026-10-29 entry, which records what 3a decided and measured, and the index's lazy load, which Derek chose on 2026-10-29 over loading it with `Lexicon.default()`; "Where the provenance lives at runtime" below describes it). **3b landed 2026-10-29**, the choice: `Locale.languages`, its tag selection, turning on, prefer and the shape clash of `may_be_initial`, the derived deviation surface, `snapshot.py --languages`, and mechanisms.md#LOCALE-PACKS-PURE-DATA amended (decisions/vocabulary-model's 2026-10-29 phase 3b entry, which records what it decided, measured and left open). **3c landed 2026-10-29**, the eight pack-shaped exclusions (cs `ml`, hr, sk and sl `st` and `ml`, fr `père`) written as `generation` readings `default = false`, the default's compile unchanged (decisions/vocabulary-model's 2026-10-29 phase 3c entry, which records what each choice now moves). **3d landed 2026-10-30**, the first language pack, `locales.NE`, a choice of `ne` turning on `given` readings of `श्री`, `guru` and `गुरु` written `default = false` in `ne.toml`, its deviations derived rather than declared, and `local_corpora.py alike --languages` for the pair measure (decisions/vocabulary-model's 2026-10-30 phase 3d entry, which records what it moves and the one defect left open). **3f landed 2026-10-30**, the Czech, Slovak, Slovenian and Croatian packs, `locales.CS`, `SK`, `SL` and `HR`, one `Locale` per language as the API section below has it, each, like `NE`, a choice of its one tag and nothing else: 3c's generations, cs and pl `pan` and cs `roz` converted from exclusions, and the Czech and Slovak academic titles and post-nominals, all readings written `default = false`, the default's compile unchanged (decisions/vocabulary-model's 2026-10-30 phase 3f entry, which records what each pack moves and settles 3c's three open questions). **3h landed 2026-10-30**, the packs `ga` (`nic`, `mhic`, `mic`, `uí`), `gd` (`nic`) and `pt-BR` (`sb`), each a choice with no fragment and no patch, its deviations derived; `F` and `N` stay excluded, the pack included, measured wrong more often than right in the filings that write them (decisions/vocabulary-model's 2026-10-30 phase 3h entry). **3e landed 2026-10-30**, the packs `hi` (`smt`, `shrimati`, `shriyut`) and `bn` (`smt`, `shrimati`, `adhyapak`), each a choice alone; `shri` stays excluded and `sushri` unwritten, each opening given names more often than it is the title (decisions/vocabulary-model's 2026-10-30 phase 3e entry). **3g landed 2026-10-30**, the packs `es` (the courtesy, academic and professional titles that lead a name, `lic`, `licenciado`, `doctora`, `profesora`, `mtro`, `doña` among them, and the Spanish given-name list as a lexicon fragment) and `pt` (`doutor`, `doutora`, `professora`, `profa`), the titles written `default = false`; `don`, `dona`, `dom` and `maestro` stay excluded, `sr` keeps its generation under both, and 3g's connective was declined on measurement (decisions/vocabulary-model's 2026-10-30 phase 3g entry). **Strict was redefined on 2026-10-30** as file-strict, the chosen files and native-script vocabulary only, needing no 3i; the rest of its API is proposed and the definition measured below, nothing built (decisions/vocabulary-model's 2026-10-30 #753, file-strict entry). The record is decisions/vocabulary-model's 2026-10-27 #753 entry; this section describes its items. Phase 3 is 3.0 work (re-planned 2026-10-28): 2.8 is the last 2.x feature release, and 3.0 ships phases 3 and 4 with the facade's removal. Packs stay opt-in, so a caller who chooses no language parses as the default does, and nothing in this phase moves the default's parses. One standing text is amended with it, in 3b's PR, which builds the selection: mechanisms.md#LOCALE-PACKS-PURE-DATA's contract that a pack is add-only and declares its deviations in its docstring (a pack adds, and selects among the default's readings by language, its deviations derived for a vocabulary pack). decisions/3-0-reevaluations' line that removing base vocabulary through a pack is part of 3.0's pack design is carried out rather than moved. +Status: **agreed** (proposed 2026-10-27, #753; Derek accepted every item on 2026-10-28, six and then the other six); **3a landed 2026-10-29**, the reading index, the compiler moved into the package and `default = false` on name readings (decisions/vocabulary-model's 2026-10-29 entry, which records what 3a decided and measured, and the index's lazy load, which Derek chose on 2026-10-29 over loading it with `Lexicon.default()`; "Where the provenance lives at runtime" below describes it). **3b landed 2026-10-29**, the choice: `Locale.languages`, its tag selection, turning on, prefer and the shape clash of `may_be_initial`, the derived deviation surface, `snapshot.py --languages`, and mechanisms.md#LOCALE-PACKS-PURE-DATA amended (decisions/vocabulary-model's 2026-10-29 phase 3b entry, which records what it decided, measured and left open). **3c landed 2026-10-29**, the eight pack-shaped exclusions (cs `ml`, hr, sk and sl `st` and `ml`, fr `père`) written as `generation` readings `default = false`, the default's compile unchanged (decisions/vocabulary-model's 2026-10-29 phase 3c entry, which records what each choice now moves). **3d landed 2026-10-30**, the first language pack, `locales.NE`, a choice of `ne` turning on `given` readings of `श्री`, `guru` and `गुरु` written `default = false` in `ne.toml`, its deviations derived rather than declared, and `local_corpora.py alike --languages` for the pair measure (decisions/vocabulary-model's 2026-10-30 phase 3d entry, which records what it moves and the one defect left open). **3f landed 2026-10-30**, the Czech, Slovak, Slovenian and Croatian packs, `locales.CS`, `SK`, `SL` and `HR`, one `Locale` per language as the API section below has it, each, like `NE`, a choice of its one tag and nothing else: 3c's generations, cs and pl `pan` and cs `roz` converted from exclusions, and the Czech and Slovak academic titles and post-nominals, all readings written `default = false`, the default's compile unchanged (decisions/vocabulary-model's 2026-10-30 phase 3f entry, which records what each pack moves and settles 3c's three open questions). **3h landed 2026-10-30**, the packs `ga` (`nic`, `mhic`, `mic`, `uí`), `gd` (`nic`) and `pt-BR` (`sb`), each a choice with no fragment and no patch, its deviations derived; `F` and `N` stay excluded, the pack included, measured wrong more often than right in the filings that write them (decisions/vocabulary-model's 2026-10-30 phase 3h entry). **3e landed 2026-10-30**, the packs `hi` (`smt`, `shrimati`, `shriyut`) and `bn` (`smt`, `shrimati`, `adhyapak`), each a choice alone; `shri` stays excluded and `sushri` unwritten, each opening given names more often than it is the title (decisions/vocabulary-model's 2026-10-30 phase 3e entry). **3g landed 2026-10-30**, the packs `es` (the courtesy, academic and professional titles that lead a name, `lic`, `licenciado`, `doctora`, `profesora`, `mtro`, `doña` among them, and the Spanish given-name list as a lexicon fragment) and `pt` (`doutor`, `doutora`, `professora`, `profa`), the titles written `default = false`; `don`, `dona`, `dom` and `maestro` stay excluded, `sr` keeps its generation under both, and 3g's connective was declined on measurement (decisions/vocabulary-model's 2026-10-30 phase 3g entry). **Strict was redefined on 2026-10-30** as file-strict, the chosen files and native-script vocabulary only, needing no 3i, and the definition measured below (decisions/vocabulary-model's 2026-10-30 #753, file-strict entry). **3j landed 2026-10-31**, file-strict built as `parser_for(..., strict=True)`, with its four open API points decided that day: a strict call choosing no language raises, no `locales.choose`, the words strict ungates derived and held by a test, and a strict choice accepting a file that holds no reading (decisions/vocabulary-model's 2026-10-31 #753, phase 3j entry, which records the ORCID table re-run on the built code). The record is decisions/vocabulary-model's 2026-10-27 #753 entry; this section describes its items. Phase 3 is 3.0 work (re-planned 2026-10-28): 2.8 is the last 2.x feature release, and 3.0 ships phases 3 and 4 with the facade's removal. Packs stay opt-in, so a caller who chooses no language parses as the default does, and nothing in this phase moves the default's parses. One standing text is amended with it, in 3b's PR, which builds the selection: mechanisms.md#LOCALE-PACKS-PURE-DATA's contract that a pack is add-only and declares its deviations in its docstring (a pack adds, and selects among the default's readings by language, its deviations derived for a vocabulary pack). decisions/3-0-reevaluations' line that removing base vocabulary through a pack is part of 3.0's pack design is carried out rather than moved. **Phase 4's shape questions came first.** The output model decides what a pack can express: a patronymic field, a second surname, credentials apart from generation suffixes (#326), and what `middle` means (#704). A pack reading of a kind with no field to land in is a reading nothing can surface, so those questions were to be settled before or with phase 3's mechanism, by the same shape-before-value rule, and the output model was agreed on 2026-10-28 (decisions/vocabulary-model's 2026-10-28 #754 entry): the seven roles stay, and `generation`, `credentials`, `patronymic` and `surname2` are views read from `Token.reading` or rules/O6's tag. Of the work order below, 3a (the reading index and the shared compiler) and 3c (the pack-shaped exclusions converted) are independent of the output model: the index carries whatever kinds the files hold, and 3c moves data between two encodings that compile alike. 3b's selection semantics are independent too, and its API's dependency, what `Locale` and `Policy` look like once the facade is gone, is answered (below). The packs depend on the output model in part, each on the phase 4 sub-phase that builds its view: `cs`'s generations and `pt-BR`'s agnomens on 4b's `generation` (3f and 3h), `es` and `pt` on 4d's `surname2` and 4g's compound given name (3g); `ne`'s readings do not. @@ -194,7 +194,7 @@ Everything a pack does is one operation: **the chosen languages select which rea | turn on | a chosen language's `default = false` readings join the pool | cs `ml` (*mladší*, once 3c makes it a reading): `Jan Novák ml.` reads family `ml.` today; under `cs`, suffix `ml.` | | prefer | on a word where a chosen language holds a vocabulary reading, other languages' vocabulary readings leave; `mul.toml`'s readings never do, being every language's | `Haddad, Abd` reads suffix `Abd` and no given name today; under `ar` the bound given reading stays and the en credential leaves; cs `st`: `Jiří Dienstbier st.` reads title `st.` today (rules/H5), and under `cs` its generation reading joins and the Saint readings leave, suffix `st.`, reported as suffix-or-name since en's `given` reading of `st` still gates it | | prefer, against a shape | a clash a property records with a SHAPE reading is settled for the chosen language's own reading, by decision 20: `may_be_initial` does not hold on it | `ANA SILVA E SOUZA` reads middle `SILVA E` with a conjunction-or-initial report today; under `pt` the conjunction holds, family `SILVA E SOUZA` | -| strict (file-strict: the definition decided 2026-10-30, its API proposed, below) | every reading outside the chosen files and native-script vocabulary leaves, and with it the markers it derives | `Do Van Minh` under `FAMILY_FIRST` reads given `Van Minh` today, Dutch `van` chaining the Vietnamese middle name Văn; under strict `vi` (simulated: `vi` cannot be chosen until vi.toml holds a reading), given `Van`, middle `Minh`; an `nl` `in` turned on, with Korean In declared as a `given` reading in ko.toml, is a particle that may be given under `nl` and never-given under strict `nl` | +| strict (file-strict: the definition decided 2026-10-30, built 2026-10-31, below) | every reading outside the chosen files and native-script vocabulary leaves, and with it the markers it derives | `Do Van Minh` under `FAMILY_FIRST` reads given `Van Minh` today, Dutch `van` chaining the Vietnamese middle name Văn; under strict `vi`, given `Van`, middle `Minh` (a plain `vi` choice is refused until vi.toml holds a reading); an `nl` `in` turned on, with Korean In declared as a `given` reading in ko.toml, is a particle that may be given under `nl` and never-given under strict `nl` | The third row answers decisions/3-0-reevaluations' "a pack cannot remove `e`": `may_be_initial` records a clash between a conjunction and the initial a letter spells, and the initial is a shape reading, so a pack that prefers Portuguese settles it, as decision 7 says choosing a language does and decision 20 permits. It reaches every reading the property stands on, pt and it `e` and ca and pl `i`, the Italian one unjudged by the inventory; and it forecloses the other direction of the 2026-08-18 bullet there, an `nl` pack marking `y` as an initial, which waits with the `fru` question below. Two parts of the mechanism are new beside decision 6: the exemption of `mul.toml` from prefer (under `ca`, `i` keeps its mul generation), and strictness as a property of the whole choice. No property is toggled per pack, and no reading is written twice. @@ -217,7 +217,7 @@ The 2 broken are pairs whose romanization opens with a listed title, which reads ### Strict: the chosen files and nothing else -Status: **the definition decided 2026-10-30; the API beyond 2026-10-28's `strict=True` argument proposed, and the definition measured, on 2026-10-31** (decisions/vocabulary-model's 2026-10-30 #753, file-strict entry, which holds every measurement below with its recipe). Nothing here is built. +Status: **the definition decided 2026-10-30 and measured on 2026-10-31; the API beyond 2026-10-28's `strict=True` argument decided, and the whole built, on 2026-10-31 (3j)** (decisions/vocabulary-model's 2026-10-30 #753, file-strict entry, which holds every measurement below with its recipe, and its 2026-10-31 #753, phase 3j entry, which records the build and the measurement re-run on it). **File-strict.** A strict choice compiles only the chosen languages' files, each tag selecting its own and its ancestors' as any choice does, plus native-script vocabulary, and nothing else. Native-script vocabulary stays on because it selects itself by script (the CJK honorifics, the Cyrillic and Devanagari titles; the Korean surname list stays too, but because it is outside the per-word model and so outside any choice's compile, not by this filter); a native-script reading is one whose word has a letter and no Latin letter, read from the word as the script always is. `mul.toml` and `und.toml` are not in a strict choice. A word with no reading in what is compiled is read by position and shape alone: under strict `vi`, `do` is in no chosen file (en.toml and pt.toml hold it) and is a name word wherever it stands, while the shape rules still act, so `Dr. Minh Do` keeps title `Dr.` by its period (rules/H2) and in `Minh Do, M.D., Ph.D.` the part past the second comma stays the suffix, `Ph.D.`, holding no name word (rules/C2), but the first credential becomes the given name (given `M.D.`, family `Minh Do`), as `Minh Do, PhD` reads given `PhD`: stacked credentials do not survive. For a credential, the harm that made name-strict wait does not arise today: measured 2026-10-31, no strict choice of one tag ungates a credential acronym, because each credential's gating name reading sits in the credential's own file, so the two leave together (`Minh Do` stays family `Do`). That holds only while the remedy below adds name readings to more files and never moves one: moving `do`'s Đỗ readings from en.toml to vi.toml would ungate `do`'s credential under strict `en`. For the other kinds it can: where a chosen file holds the vocabulary reading and only a dropped file the gating name reading, strict keeps the reading ungated ("The derived markers under strict" below; strict `pt` reads `Do Van Minh` family `Do Van Minh`, the harm item 4 of the 2026-10-27 entry named). Strict still needs no 3i, because under file-strict that case is remedied by data, a name reading written in the chosen file whose names carry the word, not by an attribute naming bearers' languages. @@ -226,7 +226,7 @@ Two consequences were accepted with the definition: - **`mul.toml` stays as it is, and under file-strict it is one more language's file.** Its entries each name several languages (`doctor` four, `docent` six, `professor` English, German, Dutch, Portuguese and the Scandinavian languages), or "the European languages" (`dr`, `prof`), or carry no note (the numerals `2` and `i` to `v`, `&`), and `ing`, the Czech and Slovak engineering degree, sits there too. Open: entries may later move between en.toml and mul.toml, or into the files of the languages that write them, so that strict `en` can shed problem words; which words those are is not yet known. - **Strict `vi` drops English credentials, accepted in principle.** Vietnamese writes its degrees in Vietnamese and before the name (`TS.`, `ThS.`, `GS.`, `PGS.`, `BS.`, stacked as `PGS.TS.`) and has no generational suffix, so `PhD` and `MD` come with English-context writing, where the caller chooses `en` too. What this costs is measured below, not assumed. -**What it does, measured** (2026-10-31 on master `b577e9a8`, Python 3.11, the cached local corpora only). The simulation compiles a strict choice through the shipped compiler, `_reading_index.choose` and `compile_readings` over the index rows of the chosen tags and the native-script rows, and applies it to the default lexicon as a choice's delta is applied; a pack's policy patch and fragment apply as `parser_for` applies them. Each ORCID country's capped 20,000 records (12,962 for SK, 17,541 for NP) are scored box by box with `local_agreement.read`, leaving out the records whose given box opens with a title word (`Dr`, `Prof`, `Mr` and their kin; 1,717 for IN, 437 for NP, under 70 elsewhere), since ORCID has no title box and a holder who typed `Dr` into the given box scores a fix whenever `dr` leaves the vocabulary. "choice" is the shipped pack where one carries the language, else a bare choice of the tag; "strict + en" adds en.toml to the strict choice. Fixed and broken are against the default. +**What it does, measured** (2026-10-31 on master `b577e9a8`, Python 3.11, the cached local corpora only). The simulation compiles a strict choice through the shipped compiler, `_reading_index.choose` and `compile_readings` over the index rows of the chosen tags and the native-script rows, and applies it to the default lexicon as a choice's delta is applied; a pack's policy patch and fragment apply as `parser_for` applies them. Each ORCID country's capped 20,000 records (12,962 for SK, 17,541 for NP) are scored box by box with `local_agreement.read`, leaving out the records whose given box opens with a title word (`Dr`, `Prof`, `Mr` and their kin; 1,717 for IN, 437 for NP, under 70 elsewhere), since ORCID has no title box and a holder who typed `Dr` into the given box scores a fix whenever `dr` leaves the vocabulary. "choice" is the shipped pack where one carries the language, else a bare choice of the tag; "strict + en" adds en.toml to the strict choice. Fixed and broken are against the default. Re-run on the built code (3j, `parser_for(pack, strict=True)` in place of the simulation) on 2026-10-31, every cell of the table matched. | Country | default | choice | strict | strict + en | |---|---|---|---|---| @@ -275,14 +275,14 @@ How to read it: Nine of the ten rows lose a bare `dr` and three `prof` (`Dr.` stays a title by its period; the pt rows' records write neither often enough to rank), and every English-context population the salutations, so a strict choice made for real data is in practice strict of the language and `en`, or needs those words in its own file; this is the mul.toml Open above. -**Files that hold no readings.** hu.toml, kk.toml, kok.toml, vi.toml and zu.toml hold only exclusions (vi.toml's are `ông` and `bà`), so `selected` refuses each tag, and a strict `vi` needs readings in vi.toml before it can be asked for (the simulation bypasses the refusal). 26 further files hold no Latin-script vocabulary reading (am, as, be, bg, el, gu, ja, km, kn, ko, lo, mk, ml, mr, my, ne, or, sd, shn, si, ta, te, th, ti, uk, zh), so strict of one of them is the native-script pool plus that file's name readings: strict `ko` is the native-script vocabulary and nothing Latin. +**Files that hold no readings.** hu.toml, kk.toml, kok.toml, vi.toml and zu.toml hold only exclusions (vi.toml's are `ông` and `bà`), so a plain choice of one is refused, changing nothing, while a strict one is accepted (decided 2026-10-31): under strict, a file holding no reading is not inert, every Latin-script reading leaving, so strict `vi` is the native-script vocabulary and nothing else until vi.toml holds readings. 26 further files hold no Latin-script vocabulary reading (am, as, be, bg, el, gu, ja, km, kn, ko, lo, mk, ml, mr, my, ne, or, sd, shn, si, ta, te, th, ti, uk, zh), so strict of one of them is the native-script pool plus that file's name readings: strict `ko` is the native-script vocabulary and nothing Latin. -**The derived markers under strict.** A marker derived from a name reading leaves with the reading it gates where both sit in a dropped file, and stays where both sit in a chosen one. It does not hold in general: where the vocabulary reading is in a chosen file and the name reading that gates it only in a dropped one, strict keeps the reading and drops its gate. The vocabulary holds 16 such words, 17 readings (`st` both as a particle and as a generation), 27 word-and-tag pairs over 13 tags (2026-10-31, each strict choice of one tag compared with the plain choice of the same tag): `cheikh` and `sultan` under `ar` (their `given` readings in fr.toml and tr.toml), `la`, `marquis`, `marquise` and `st` under `fr`, `mac` under `ga` and `gd`, `nic` under `gd` (its name readings in ga.toml), `del`, `la`, `san` and `santa` under `it`, `bin` and `sultan` under `ms`, `den` and `tho` under `nl`, `da`, `do` and `santa` under `pt` and `pt-BR`, and the generation `st` the choice turns on under `cs`, `hr`, `sk` and `sl` (its `given` reading in en.toml). Each becomes a never-given particle, an unconditional title or a generation that reports nothing (under the plain `cs` choice `Novák, St` reads given `St` and reports suffix-or-name; under strict `cs`, suffix `St` and no report; `Nic Smith` under `gd`, given `Nic` reported particle-or-given, and under strict `gd` family `Nic Smith`): under strict `pt`, `Da Costa` reads family `Da Costa` where the default and the `pt` choice read given `Da`; under strict `ar`, `Sultan Ahmed Khan` reads title `Sultan`, where the default reads given `Sultan`. Some of these are what strict is for (a Portuguese `Do` is the particle), and some are the file-placement fact that sank name-strict, at small scale: Sultan is an Arabic given name whose reading sits in tr.toml. Under file-strict the remedy is data, a name reading written in each file whose names carry it, as a word already sits in the file of every language whose names it occurs in; the list is derivable from the index, as a choice's changed words are, so a test can hold it to a reviewed set. +**The derived markers under strict.** A marker derived from a name reading leaves with the reading it gates where both sit in a dropped file, and stays where both sit in a chosen one. It does not hold in general: where the vocabulary reading is in a chosen file and the name reading that gates it only in a dropped one, strict keeps the reading and drops its gate. The vocabulary holds 16 such words, 17 readings (`st` both as a particle and as a generation), 27 word-and-tag pairs over 13 tags (2026-10-31, each strict choice of one tag compared with the plain choice of the same tag): `cheikh` and `sultan` under `ar` (their `given` readings in fr.toml and tr.toml), `la`, `marquis`, `marquise` and `st` under `fr`, `mac` under `ga` and `gd`, `nic` under `gd` (its name readings in ga.toml), `del`, `la`, `san` and `santa` under `it`, `bin` and `sultan` under `ms`, `den` and `tho` under `nl`, `da`, `do` and `santa` under `pt` and `pt-BR`, and the generation `st` the choice turns on under `cs`, `hr`, `sk` and `sl` (its `given` reading in en.toml). Each becomes a never-given particle, an unconditional title or a generation that reports nothing (under the plain `cs` choice `Novák, St` reads given `St` and reports suffix-or-name; under strict `cs`, suffix `St` and no report; `Nic Smith` under `gd`, given `Nic` reported particle-or-given, and under strict `gd` family `Nic Smith`): under strict `pt`, `Da Costa` reads family `Da Costa` where the default and the `pt` choice read given `Da`; under strict `ar`, `Sultan Ahmed Khan` reads title `Sultan`, where the default reads given `Sultan`. Some of these are what strict is for (a Portuguese `Do` is the particle), and some are the file-placement fact that sank name-strict, at small scale: Sultan is an Arabic given name whose reading sits in tr.toml. Under file-strict the remedy is data, a name reading written in each file whose names carry it, as a word already sits in the file of every language whose names it occurs in; the list is derived from the index, as a choice's changed words are (`_reading_index.strict_ungated`), and a test holds it to the reviewed set above (decided 2026-10-31). -**The API (the `strict=True` argument decided 2026-10-28; the rest proposed, not decided).** +**The API (the `strict=True` argument decided 2026-10-28; the rest decided and built 2026-10-31).** -- **How a caller asks: an argument of `parser_for`, `strict=True`, over the whole choice, decided 2026-10-28 and unchanged by file-strict.** What is proposed is the rest. A `Locale.strict` field, the alternative, is not recommended (proposed, like the rest): the choice is the union of the packs' languages, and a strict pack stacked with a plain one has no meaning, a plain choice keeping every file. A strict call whose packs choose no language (`parser_for(locales.ZH, strict=True)`) would keep only native-script vocabulary; it should raise, naming the packs and asking for a language, rather than strip a caller's Latin vocabulary in silence. A strict choice is one call's, as a choice is (decisions/vocabulary-model's chained-choices entry): as a base it records nothing, so a later plain choice applies its delta to an already stripped lexicon. -- **Shipped `Locale` constants: keep them for packs that carry something, and add a constructor for the rest (recommended).** A constant per language file would be 78 constants, one per tag a file holds a reading of beside `mul` and `und` (2026-10-31), most of them a tag and nothing else. Today a caller writes `Locale(code="de", lexicon=Lexicon.empty(), languages=frozenset({"de"}))`, which is verbose for a choice and nothing more. A `locales.choose("vi", "en")` returning that Locale (an addition), validated by `selected`, covers every language with a file; the constants stay for the language-choice packs with a policy patch, a fragment, or a measured docstring (NE, CS, SK, SL, HR, GA, GD, PT_BR, HI, BN, ES, PT), and ZH, JA, RU and TR_AZ, which choose no language and carry a fragment or a policy patch, stay as they are. Folding a bare tag into `locales.get` was weighed and is not recommended: `get` returns a shipped pack, and a choice it built would be one no test has measured. +- **How a caller asks: an argument of `parser_for`, `strict=True`, over the whole choice, decided 2026-10-28 and unchanged by file-strict.** No `Locale.strict` field: the choice is the union of the packs' languages, and a strict pack stacked with a plain one has no meaning, a plain choice keeping every file. A strict call whose packs choose no language (`parser_for(locales.ZH, strict=True)`) would keep only native-script vocabulary, so it raises `ValueError`, naming the packs and offering a pack that chooses a language, rather than strip a caller's Latin vocabulary in silence (decided 2026-10-31). A strict choice is one call's, as a choice is (decisions/vocabulary-model's chained-choices entry): as a base it records nothing, so a later plain choice applies its delta to an already stripped lexicon. +- **Shipped `Locale` constants: keep them for packs that carry something, and no constructor for the rest (decided 2026-10-31).** A constant per language file would be 78 constants, one per tag a file holds a reading of beside `mul` and `und` (2026-10-31), most of them a tag and nothing else. A caller writes `Locale(code="de", lexicon=Lexicon.empty(), languages=frozenset({"de"}))`, verbose for a choice and nothing more; a `locales.choose("vi", "en")` returning that Locale was proposed and is left out, the bare spelling being the one the docs teach. The constants stay for the language-choice packs with a policy patch, a fragment, or a measured docstring (NE, CS, SK, SL, HR, GA, GD, PT_BR, HI, BN, ES, PT), and ZH, JA, RU and TR_AZ, which choose no language and carry a fragment or a policy patch, stay as they are. Folding a bare tag into `locales.get` was weighed and is not recommended: `get` returns a shipped pack, and a choice it built would be one no test has measured. - **Turned-on readings:** a strict choice turns on the chosen files' `default = false` readings, as a choice does; another file's off reading stays off, a native-script one included. - **A pack's `[rules]` patch and its fragment** are untouched: the patch is policy, and the fragment is unioned after the strict compile, so `locales.ES`'s given-name list and `locales.ZH`'s surnames stay (measured: the 1,000 entries of ES's list under strict `es`). A fragment is the pack's own data, which strict keeps by definition. - **`Lexicon.given_names` and `Lexicon.surnames`** are not compiled from the files, so no choice touches them, strict included: the Korean surname list, outside the per-word model, stays under every strict choice (measured). @@ -291,7 +291,7 @@ Nine of the ten rows lose a bare `dr` and three `prof` (`Dr.` stays a title by i **Candidate next steps, not in this design's PR:** -- **Vietnamese degree prefixes as vi.toml readings** (`TS.`, `ThS.`, `GS.`, `PGS.`, `BS.`, and the stacked `PGS.TS.`), which would also give vi.toml its first readings and make a `vi` choice possible. Each needs its collisions and bearer counts checked first, per nameparser/vocab/AGENTS.md: `Ts`, `Gs` and `Bs` are short enough to be initials or other languages' words. +- **Vietnamese degree prefixes as vi.toml readings** (`TS.`, `ThS.`, `GS.`, `PGS.`, `BS.`, and the stacked `PGS.TS.`), which would also give vi.toml its first readings and make a plain `vi` choice possible. Each needs its collisions and bearer counts checked first, per nameparser/vocab/AGENTS.md: `Ts`, `Gs` and `Bs` are short enough to be initials or other languages' words. - **`ông` and `bà` stay excluded.** Ông is a Vietnamese surname, a clash within Vietnamese that strict does not settle. - **The candidates that waited for strict become buildable as data.** 3h's `nl` prepositions: `in` and `to` turned on `default = false` in nl.toml, with Korean In and Vietnamese To declared as `given` readings in ko.toml and vi.toml, are possibly-given particles under the plain `nl` choice and never-given under strict `nl`. The plain choice still carries the cost item 4 of the 2026-10-27 entry recorded: the marker acts at the head only, so `Park In Hwan` reads family `In Hwan` and `Nguyen To Nga` family `To Nga` (re-run 2026-10-31 with `particles` and `particles_ambiguous` holding `in` and `to`), and a follow-up must not ship them under the plain `nl` choice without settling it. 3e's `sushri`: a hi.toml salutation `default = false` with the Odia given name declared in or.toml is a title only inside a title phrase under the plain `hi` choice (rules/H9) and a title under strict `hi`. The build has not been asked whether it accepts either; that is each candidate's own PR. - **The 16 strict-ungated words** each read, and a name reading added to the chosen file where that language's names carry the word. @@ -310,7 +310,7 @@ A choice is applied as a delta against the default's compile, so it composes wit ### The API and tags -This rested on `Locale` and `Policy` keeping their 2.x shape after the facade's removal, and the 2026-10-28 3.0 freeze answers it: both keep it, with additions. A choice is carried by `Locale`, which gains a `languages` field, a set of BCP 47 tags. `parser_for` unions the `languages` of the packs it is given, so the choice is order-independent while their policy patches still fold in order. Strict is a property of the whole choice, not of one pack, and an argument: `parser_for(locales.NE, locales.HI, strict=True)` (file-strict, the rest of its API proposed in "Strict: the chosen files and nothing else" above). The shipped language packs are `Locale` constants like the four today (`locales.NE`, `locales.get("ne")`), each with an empty lexicon fragment (what a fragment holds once 3.0 configures vocabulary by reading is decisions/vocabulary-model's the 2026-10-28 #754 entry on the `Lexicon` surface); a code keeps `Locale`'s `[a-z0-9_]+`, so `pt-BR` is `locales.PT_BR`. +This rested on `Locale` and `Policy` keeping their 2.x shape after the facade's removal, and the 2026-10-28 3.0 freeze answers it: both keep it, with additions. A choice is carried by `Locale`, which gains a `languages` field, a set of BCP 47 tags. `parser_for` unions the `languages` of the packs it is given, so the choice is order-independent while their policy patches still fold in order. Strict is a property of the whole choice, not of one pack, and an argument: `parser_for(locales.NE, locales.HI, strict=True)` (file-strict, its API in "Strict: the chosen files and nothing else" above). The shipped language packs are `Locale` constants like the four today (`locales.NE`, `locales.get("ne")`), each with an empty lexicon fragment (what a fragment holds once 3.0 configures vocabulary by reading is decisions/vocabulary-model's the 2026-10-28 #754 entry on the `Lexicon` surface); a code keeps `Locale`'s `[a-z0-9_]+`, so `pt-BR` is `locales.PT_BR`. A chosen tag selects its own file and its ancestors', never its descendants': `pt-BR` selects `pt-BR.toml` and `pt.toml`, and `pt` selects only `pt.toml`, so a caller in Portugal does not turn on the Brazilian agnomens (#843 left the single-letter `F` and `N` to a `pt-BR` pack). No macrolanguage table is read: Malay and Indonesian are separate tags, though BCP 47 counts Indonesian within the Malay macrolanguage, and a caller with names in both chooses both. The Czech, Slovak, Slovenian and Croatian files each hold the words they share, so each language is a pack of its own. Script subtags need no matching rule yet, since no file carries one. @@ -346,7 +346,7 @@ Shape first, then the packs, each one PR measured on its own base, as phase 2's | 3g | `es` and `pt`: the occupational titles and `Don`/`Doña`/`Dona`, with a connective that counts only inside a title run, its own rule; `pt` also settles the one-case `e`. Landed with the titles that take no complement, `Doña` alone of the courtesy titles, the one-case `e` settled by the `pt` choice (3b's rule), and no connective, declined on measurement (the 2026-10-30 phase 3g entry) | 3b, 4d, 4g | | 3h | `ga` and `gd` (`nic`, `mhic`, `mic`, `uí`), `pt-BR` (`F`, `N`, `sb`); `nl`'s prepositions wait with strict. Landed with `mhic` and `mic` in `ga` only and `pt-BR` as `sb` alone (the 2026-10-30 phase 3h entry) | 3b, 4b | | 3i | the languages of each name reading's bearers, an attribute compiling to nothing, filled word by word from per-country shares and whole-box counts, for name-strict to read; set aside 2026-10-30 with name-strict, file-strict needing none, and left with no consumer | — | -| 3j | file-strict: `parser_for(..., strict=True)` (the argument decided 2026-10-28, the definition 2026-10-30) compiling only the chosen files and native-script vocabulary; proposed 2026-10-31 and not decided: refusing a strict call that chooses no language, `locales.choose`, and a derived list of the words strict ungates, each strict choice compared with the plain choice of the same tags, never with the default's markers (which miss a reading the choice turns on), held by a test; and `_reading_index`'s `UNCHOOSABLE` comment and `selected`'s refusal text, which say no choice removes a reading of `mul` or `und`, which strict does. The candidates that waited on strict follow it, each in its own PR | 3b; a `vi` choice also needs readings in vi.toml | +| 3j | file-strict: `parser_for(..., strict=True)` (the argument decided 2026-10-28, the definition 2026-10-30) compiling only the chosen files and native-script vocabulary. Landed 2026-10-31 with the API decided that day: a strict call choosing no language raises; no `locales.choose`; a derived list of the words strict ungates, each strict choice compared with the plain choice of the same tags, never with the default's markers (which miss a reading the choice turns on), held by a test; a strict choice accepting a file that holds no reading, which a plain one refuses; `_reading_index`'s `UNCHOOSABLE` comment and `selected`'s refusal text saying a strict choice drops `mul` and `und`; and the CLI's `--strict` (the 2026-10-31 phase 3j entry). The candidates that waited on strict follow it, each in its own PR | 3b; a plain `vi` choice also needs readings in vi.toml | **Declined 2026-10-30 (the phase 3g entry in decisions/vocabulary-model):** 3g's connective, which needed a design of its own, not decided with the work order, because the pair that separates it is a count: `Profesora de Historia Ana López` should read title `Profesora de Historia`, while `Doctora de la Vega`, the same title, preposition and following word, has no name behind it and must keep family `de la Vega`. With `profesora` and `ministro` added as titles and no rule, the first reads family `de Historia Ana López` (the rules/P1 fold), and `Ministro da Fazenda Carlos Souza` reads given `da`. The rule must reach only readings a pack turns on, so the default's parses stay where they are. No rule was written: over 8,526,439 names of the ten local and ten committed corpora measured, no leading title and connective is a title phrase with a name behind it, the 16 that open that way being particle-led surnames and given names behind a title, the costs, a given name that is no title (`Cardeal`), one office with no name behind it and a fragment; Spanish and Portuguese write an office lowercase in apposition, and the office titles that take a complement stay out of the packs. diff --git a/docs/locales.rst b/docs/locales.rst index e3c2cb9df..8d12167f2 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -186,7 +186,7 @@ never the other way: ``pt-BR`` reads ``pt.toml`` too, and ``pt`` leaves the Brazilian words alone. No other table of related languages is read, so a caller with Malay and Indonesian names chooses ``ms`` and ``id``. Tags are compared without regard to case, and a tag is -checked in one of three ways: +checked in one of four ways: - **A malformed tag** (``pt_BR``, ``p``) raises ``ValueError`` when the :class:`~nameparser.Locale` is built. @@ -196,6 +196,10 @@ checked in one of three ways: - **A region or other subtag of a language that has a file** (``pt-PT``) is accepted and reads that language's file, there being none of its own. +- **A language whose file holds no reading yet** (``vi``, whose file + only records words kept out) raises ``ValueError``, a choice of it + changing nothing; a strict choice accepts it (`Reading one language + only`_). A pack that chooses no language leaves the base parser's vocabulary exactly as it was. @@ -235,6 +239,78 @@ credential: >>> sorted(parser_for(en, base=Parser(lexicon=mine)).lexicon.particles) ['van'] +.. _strict-choice: + +Reading one language only +^^^^^^^^^^^^^^^^^^^^^^^^^ + +A choice still reads every other language's words: it settles only the +words its own language has a reading of. Names of a single origin +written in Latin letters meet those words by accident. Many common +Korean syllables, romanized, are spelled like another language's +word: ``Do`` like the Portuguese particle, ``Bin`` like the Arabic +and Malay one, ``Ra`` like an English credential and a German academic +title. Korean's file holds no word written in Latin letters, so a +``ko`` choice settles none of them. + +``parser_for(..., strict=True)`` makes the choice strict: only the +chosen languages' files are read, each tag selecting its own and its +ancestors' as above, plus every word written in a script other than +Latin (the Korean, Japanese and Chinese honorifics, the Cyrillic and +Devanagari titles), which selects itself by its script. Every other +language's words leave, the shared ``mul`` and ``und`` files' +included, and a word no chosen file holds is read as a name word: + +.. doctest:: + + >>> ko = Locale(code="ko", lexicon=Lexicon.empty(), + ... languages=frozenset({"ko"})) + >>> korean = parser_for(ko, strict=True) + >>> parse("Kim Do Hyun").family + 'Do Hyun' + >>> korean.parse("Kim Do Hyun").middle, korean.parse("Kim Do Hyun").family + ('Do', 'Hyun') + >>> parse("Kim Min Ra").suffix + 'Ra' + >>> korean.parse("Kim Min Ra").family + 'Ra' + +Japanese ``Gen`` and Chinese ``De`` are the same case under strict +``ja`` and ``zh``: ``Gen Hoshino`` reads given ``Gen`` rather than a +title, and ``De Xu`` given ``De``, family ``Xu``. + +Strict is for data you know to be of one origin, and what it drops it +drops from every name: + +- **Other languages' words go with the accidents.** A credential + written in English (``Kim Min Jun, MD`` reads given ``MD``), a bare + ``Dr`` or ``Mr``, and an immigrant's particle from another language + are read as name words. A shape still reads as it does by default: + ``Dr.``, with its period, is a title. Where English context is part + of the data, choose ``en`` too, which brings back its words and its + costs alike: + + .. doctest:: + + >>> parser_for(ko, en, strict=True).parse("Kim Min Ra").suffix + 'Ra' + >>> parser_for(ko, en, strict=True).parse("Kim Min Jun, MD").suffix + 'MD' + +- **Strict needs a language.** A strict call whose packs choose none + (``parser_for(locales.ZH, strict=True)``) would keep only the words + in other scripts, so it raises ``ValueError`` and names a pack to + add. A pack's own word list and policy are applied after the strict + choice, untouched, and so is the Korean surname list. +- **A language whose file holds no reading yet is accepted.** + ``vi.toml`` holds none, so a plain ``vi`` choice is refused; a + strict one reads the other scripts' words and nothing in Latin + letters, which unchains Dutch ``van`` from a Vietnamese middle name + (``Do Van Minh`` reads middle ``Van``, family ``Minh``). +- **It is a delta, like a choice.** A word a strict choice drops leaves + a base parser's vocabulary even where you added it yourself; add it + again in the pack's own lexicon, which is applied after. + .. _west-south-slavic-packs: Czech, Slovak, Slovenian and Croatian names diff --git a/docs/release_log.rst b/docs/release_log.rst index 2b250dd73..969d4c4ca 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -19,6 +19,7 @@ Release Log - **Add Reading.languages, the language files that list a word the way it was read.** ``parse("Dr. Juan de la Vega III").tokens[2].reading.languages`` is ``('es', 'fr', 'it', 'nl', 'pt')`` for the particle ``de``, and ``Ambiguity.alternative`` names its files too; a reading by shape or position names none. It is not compared, and a ``Reading`` pickled by 2.7 or 2.8 loads with none. The first parse in a process that meets a listed word loads the reading index once: about 1 ms, or about 16 ms where Python writes no bytecode cache. :issue:`754`, :issue:`753` - **Add Czech, Slovak, Slovenian and Croatian packs, locales.CS, SK, SL and HR.** ``parser_for(locales.CS).parse("Jan Novák, MUDr")`` gives title ``MUDr``, given ``Jan``, family ``Novák``, where 1.4.0 through 2.8.0 gave given ``MUDr``, family ``Jan Novák``. Each pack chooses its one language and turns on words the default does not read: ``ml.`` and ``st.`` after the name in all four (``Jiří Dienstbier ml.``), the Czech and Slovak academic titles and ``DrSc.``, and Czech ``Pan`` and ``roz.``. Choosing the language drops ``st``'s Saint readings (``St Clair Smith`` reads given ``St``), which choosing ``fr`` as well keeps. ``CS`` and ``SK`` also read two family names, so ``Alena Pešová Řehořová`` gives family ``Pešová Řehořová`` and two given names read as a given name and a family name (``Petr Pavel Novák`` gives family ``Pavel Novák``); a family comma keeps both. :issue:`753` - **Add Hindi and Bengali packs, locales.HI and locales.BN, for the romanized Indic titles.** ``parser_for(locales.HI).parse("Smt Nirmala Sitharaman")`` gives title ``Smt``, given ``Nirmala``, where 1.4.0 through 2.8.0 gave given ``Smt``, middle ``Nirmala``. ``HI`` reads ``Smt``, ``Shrimati`` and ``Shriyut`` as titles and ``BN`` ``Smt``, ``Shrimati`` and ``Adhyapak``; ``Shri`` and ``Sushri`` stay name words under both, since each opens given names (``Shri Kant``, ``Sushri Sangita``). See :doc:`locales`. :issue:`345`, :issue:`753` + - **Add parser_for(..., strict=True), to read only the chosen languages' words.** ``parser_for(Locale(code="ko", lexicon=Lexicon.empty(), languages=frozenset({"ko"})), strict=True).parse("Kim Do Hyun")`` gives middle ``Do``, family ``Hyun``, where the default, as in 1.4.0 through 2.8.0, and a plain ``ko`` choice give family ``Do Hyun``, ``Do`` read as the Portuguese particle. Only the chosen languages' vocabulary files are read, with the words written in other scripts; every other language's words leave, the shared ones (``Dr``, ``Prof``) and English credentials included, so ``Kim Min Jun, MD`` gives given ``MD`` unless ``en`` is chosen too. A strict call that chooses no language raises ``ValueError``. ``python -m nameparser`` gains ``--strict``, and ``--locale`` takes a language tag no pack has (``--locale ko``). See :doc:`locales`. :issue:`753` - **Add --order to python -m nameparser, to read a name in another order.** ``python -m nameparser --order family-first "Kovács János Péter"`` gives family ``Kovács``, given ``János``, where the command, in 2.0.0 through 2.8.0, read only the default order and gave family ``Péter``. It takes ``given-first``, ``family-first`` or ``family-first-given-last``; a name in Han, Hangul or kana keeps its script's order. :issue:`983` - **Divide Japanese names under python -m nameparser --locale ja when nameparser[ja] is installed.** ``python -m nameparser --locale ja "山田太郎"`` gives family ``山田``, given ``太郎``, where 2.1.0 through 2.8.0 left family ``山田太郎`` and warned. Without the extra it reads as before and prints how to install it, in place of the warning; ``parser_for(locales.JA)`` still needs ``segmenter=locales.ja_segmenter()``. :issue:`984` diff --git a/docs/usage.rst b/docs/usage.rst index 33846ec80..cc9edabbb 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1451,7 +1451,17 @@ form to pipe somewhere: {"title": "", "given": "John", "middle": "", "family": "Doe", "suffix": "", "nickname": "", "maiden": ""} Add ``--locale`` to parse with a locale pack (for example ``--locale -ru``); see :doc:`locales`. ``--order`` reads the name in another order +ru``); see :doc:`locales`. A code no pack has that is a language with +a vocabulary file (``--locale ko``) chooses that language alone. +``--strict`` reads only that language's words (see +:ref:`strict-choice`), and needs ``--locale``: + +:: + + $ python -m nameparser --locale ko --strict --json "Kim Do Hyun" + {"title": "", "given": "Kim", "middle": "Do", "family": "Hyun", "suffix": "", "nickname": "", "maiden": ""} + +``--order`` reads the name in another order — ``given-first`` (the default), ``family-first`` or ``family-first-given-last``, the three ``Policy.name_order`` values — and replaces any order a pack sets. A name written wholly in Han, From 80a3355719e421c0913f6ee2f6cc25017b46ea2d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 31 Oct 2026 17:57:55 -0700 Subject: [PATCH 3/7] Review fixes: pointers, vi wording, strict and prefer refused as data (#753, 3j) AGENTS.md named the wrong test for strict_ungated; the CLI help and usage.rst said any language with a file works plainly, where vi needs --strict; parser_for's docstring pointed only at the entry whose vi bullet 3j superseded; the RFC's ORCID cell said no file holds vi. The build now refuses prefer and strict as data, naming the caller's call, as the RFC's phase 3 API had decided, in place of 'reserved for phase 3'. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- docs/design/decisions/vocabulary-model.md | 1 + docs/design/vocabulary-model-rfc.md | 4 ++-- docs/usage.rst | 5 +++-- nameparser/__main__.py | 3 ++- nameparser/_parser.py | 3 ++- nameparser/vocab/README.md | 4 ++-- tests/v2/test_vocab.py | 6 +++++- tools/vocab/build.py | 12 +++++++++++- 9 files changed, 29 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 39410e071..b3749fc8c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -206,7 +206,7 @@ Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely o ### Vocabulary (`nameparser/vocab/`) and the configuration layer (`nameparser/config/`) -**The default vocabulary is written per word, in TOML, and compiled.** `nameparser/vocab/.toml` holds one file per BCP 47 language tag, each `[[word]]` entry a `text` and its readings keyed by kind. What a file means, how an entry is built, the kinds and properties with the set each compiles into, exclusions, bearer counts and the steps for adding a word are in `nameparser/vocab/README.md`, the reference for anyone editing the TOML; `tests/v2/test_vocab.py` holds its kind, property, derived-marker and reserved-name tables to the names `build.py` accepts and the sets it compiles, so read it rather than restating it here. The parser never reads the TOML: `tools/vocab/build.py` runs the build checks that replaced the modules' import-time asserts (its docstring numbers them), turns every reading into an index row with its file's tag, and writes the rows, and every file's tag (`FILES`, a file holding only exclusions included, for a strict choice to name), as the committed reading index `nameparser/config/_index.py` (#753, phase 3a) and, compiled, as the committed `nameparser/config/_compiled.py`, frozenset literals one entry per line; the build never reads `_index.py` back. A file a shipped pack reads may also hold a `[rules]` table, each key a `Policy` field (`patronymic_rules`, `name_order`, `two_family_names`, `two_given_names`), which the build checks against its `RULE_KEYS` and `PACK_RULES` and writes as `nameparser/config/_rules.py`; `_locale.rules_patch(tag)` turns a file's table into the `PolicyPatch` a pack carries, RU's and TR_AZ's from `ru.toml` and `tr.toml`, and the `two_family_names` of ES, CS and SK from `es.toml`, `cs.toml` and `sk.toml` (#754, 4f; decisions/vocabulary-model's 2026-10-29 4f entry). The compiler is `nameparser/_reading_index.py`'s `compile_readings`, with the kind and property tables it reads (`CATEGORY`, `BEHAVIORAL`, `GATED_WORDS`), so the build and a choice of languages compile alike: `Locale.languages` (3.0, phase 3b) is a choice of BCP 47 tags, which `parser_for` unions across its packs and applies as a delta against the default's compile, `choose` there selecting the readings (a chosen file's `default = false` ones on; on a word a chosen file holds a vocabulary reading of, other files' vocabulary readings off, save `mul.toml`'s and `und.toml`'s; a chosen conjunction's `may_be_initial` dropped; under `parser_for(..., strict=True)`, 3j, only the chosen files and native-script vocabulary compiled, through `strict_pool`), `changed_words` deriving a choice's deviation surface and `strict_ungated` the words a strict choice ungates against the plain one, which tests/v2/test_locales.py's gate reads (decisions/vocabulary-model, 2026-10-29 3b); the build keeps the checks and loads that module by path, as it loads `_lexicon.py`, and `tests/v2/test_reading_index.py` holds that the index compiled with no choice is `_compiled.py` exactly. `readings_of(text)` there is the run-time reader of the index, every reading of a word with its file's tag, loaded on first call; `languages_of(text, categories)` is the reader a parse calls, from assemble, for `Reading.languages` (3.0), so the first parse that reads a word from its `Lexicon`'s vocabulary loads the index. Each `config/` module re-exports its constants from `_compiled.py` (`TITLES = _compiled.TITLES`) under its autodoc docstring, so the public names and the docs did not move. `tests/v2/test_vocab.py` fails when `_compiled.py` is stale, and `tests/v2/test_reading_index.py` when the index is, each naming the refresh command, and `test_vocab.py` gives every build check a mutation. The model, the phasing and what was declined: decisions/vocabulary-model. +**The default vocabulary is written per word, in TOML, and compiled.** `nameparser/vocab/.toml` holds one file per BCP 47 language tag, each `[[word]]` entry a `text` and its readings keyed by kind. What a file means, how an entry is built, the kinds and properties with the set each compiles into, exclusions, bearer counts and the steps for adding a word are in `nameparser/vocab/README.md`, the reference for anyone editing the TOML; `tests/v2/test_vocab.py` holds its kind, property, derived-marker and reserved-name tables to the names `build.py` accepts and the sets it compiles, so read it rather than restating it here. The parser never reads the TOML: `tools/vocab/build.py` runs the build checks that replaced the modules' import-time asserts (its docstring numbers them), turns every reading into an index row with its file's tag, and writes the rows, and every file's tag (`FILES`, a file holding only exclusions included, for a strict choice to name), as the committed reading index `nameparser/config/_index.py` (#753, phase 3a) and, compiled, as the committed `nameparser/config/_compiled.py`, frozenset literals one entry per line; the build never reads `_index.py` back. A file a shipped pack reads may also hold a `[rules]` table, each key a `Policy` field (`patronymic_rules`, `name_order`, `two_family_names`, `two_given_names`), which the build checks against its `RULE_KEYS` and `PACK_RULES` and writes as `nameparser/config/_rules.py`; `_locale.rules_patch(tag)` turns a file's table into the `PolicyPatch` a pack carries, RU's and TR_AZ's from `ru.toml` and `tr.toml`, and the `two_family_names` of ES, CS and SK from `es.toml`, `cs.toml` and `sk.toml` (#754, 4f; decisions/vocabulary-model's 2026-10-29 4f entry). The compiler is `nameparser/_reading_index.py`'s `compile_readings`, with the kind and property tables it reads (`CATEGORY`, `BEHAVIORAL`, `GATED_WORDS`), so the build and a choice of languages compile alike: `Locale.languages` (3.0, phase 3b) is a choice of BCP 47 tags, which `parser_for` unions across its packs and applies as a delta against the default's compile, `choose` there selecting the readings (a chosen file's `default = false` ones on; on a word a chosen file holds a vocabulary reading of, other files' vocabulary readings off, save `mul.toml`'s and `und.toml`'s; a chosen conjunction's `may_be_initial` dropped; under `parser_for(..., strict=True)`, 3j, only the chosen files and native-script vocabulary compiled, through `strict_pool`), `changed_words` deriving a choice's deviation surface which tests/v2/test_locales.py's gate reads, and `strict_ungated` the words a strict choice ungates against the plain one, which tests/v2/test_reading_index.py holds to a reviewed table (decisions/vocabulary-model, 2026-10-29 3b; 2026-10-31 #753, phase 3j); the build keeps the checks and loads that module by path, as it loads `_lexicon.py`, and `tests/v2/test_reading_index.py` holds that the index compiled with no choice is `_compiled.py` exactly. `readings_of(text)` there is the run-time reader of the index, every reading of a word with its file's tag, loaded on first call; `languages_of(text, categories)` is the reader a parse calls, from assemble, for `Reading.languages` (3.0), so the first parse that reads a word from its `Lexicon`'s vocabulary loads the index. Each `config/` module re-exports its constants from `_compiled.py` (`TITLES = _compiled.TITLES`) under its autodoc docstring, so the public names and the docs did not move. `tests/v2/test_vocab.py` fails when `_compiled.py` is stale, and `tests/v2/test_reading_index.py` when the index is, each naming the refresh command, and `test_vocab.py` gives every build check a mutation. The model, the phasing and what was declined: decisions/vocabulary-model. The `config/` SETS are frozen since 2.2 (#293): there is no `.add()`/`.remove()` on any of them, so a default word list is changed at runtime by configuring a `Lexicon`, never by editing the constant, and the shipped default is changed by editing the TOML and rebuilding. **Neither dict was frozen, and `CAPITALIZATION_EXCEPTIONS` is not covered by anything else either.** `REGEXES` is a compiled-pattern table rather than vocabulary and was never in #293's scope; `CAPITALIZATION_EXCEPTIONS` is vocabulary-shaped and is a decided, in-scope exemption. So the hazard the freeze closes is still live for it: `CAPITALIZATION_EXCEPTIONS['dclinpsy'] = 'DClinPsy'` after the first parse misses the cached `Lexicon.default()`. Same advice — configure the object (`dataclasses.replace(lexicon, capitalization_exceptions=...)`). `tests/v2/test_contracts.py::test_every_vocabulary_constant_is_frozen` names both dicts as explicit exemptions rather than letting its `isinstance` filter drop them. diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index 6157edaee..3f7b900b7 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -452,6 +452,7 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) 2. **No public `locales.choose`.** A bare choice is a `Locale` with an empty lexicon and the tags, the spelling docs/usage.rst teaches. 3. **The words strict ungates are derived**, `_reading_index.strict_ungated`, each strict choice against the plain choice of the same tags and never against the default's markers, which miss a reading the choice turns on; tests/v2/test_reading_index.py holds every one-tag strict choice to a per-tag table, which is the 2026-10-30 entry's 16 words and 27 word-and-tag pairs over 13 tags exactly, so a word that joins or leaves is a reviewed diff. 4. **A strict choice accepts a tag whose file holds no reading** (hu, kk, kok, vi and zu today): under strict it is not inert, every Latin-script reading leaving. A plain choice of one is still refused, its message now saying a strict one accepts it. The reading index records every file's tag for this (`FILES` in nameparser/config/_index.py, read by `_reading_index.files`), a file holding only exclusions writing no row of `READINGS`. + - The build refuses `prefer` and `strict` as data, saying a strength is the caller's (`parser_for(..., strict=True)`), where it had said each was reserved for phase 3; the RFC's phase 3 API section had decided the message. - `_reading_index`'s `UNCHOOSABLE` comment and `selected`'s refusal text, which said no choice removes a reading of `mul` or `und`, now say a plain choice never does and a strict one drops both. - **The command line** gains `--strict`, which needs `--locale` (exit 2 without it). `--locale` falls back from a pack code to a bare choice of a language tag (`--locale ko`), and a code that is neither exits 2 listing the pack codes and the tags. Under `--strict`, a pack that chooses no language chooses its own code (`--locale zh --strict`), and one whose code is no language (`tr_az`) exits 2. These are the debug helper's, chosen with the build rather than decided as API. - **Measured: the 2026-10-30 entry's ORCID table re-run on the built code**, its recipe with `parser_for(pack, strict=True)` (and a bare `en` pack beside it for strict + en) in place of the simulation, on this phase's branch over `8438ccbb`, Python 3.11: every cell matches, 14 countries by four columns. diff --git a/docs/design/vocabulary-model-rfc.md b/docs/design/vocabulary-model-rfc.md index 3a925bf7f..742d3759a 100644 --- a/docs/design/vocabulary-model-rfc.md +++ b/docs/design/vocabulary-model-rfc.md @@ -230,7 +230,7 @@ Two consequences were accepted with the definition: | Country | default | choice | strict | strict + en | |---|---|---|---|---| -| VN | 65.48% | none: no file holds `vi` | 64.28% (+349 −589) | 64.34% (+306 −534) | +| VN | 65.48% | none: vi.toml holds no reading, so a plain `vi` choice is refused | 64.28% (+349 −589) | 64.34% (+306 −534) | | KR | 92.42% | 92.42% (+0 −0) | 92.54% (+41 −17) | 92.51% (+33 −16) | | CN | 84.54% | 84.54% (+0 −0) | 84.65% (+23 −3) | 84.61% (+16 −3) | | JP | 95.59% | 95.59% (+0 −0) | 95.64% (+30 −20) | 95.54% (+11 −20) | @@ -314,7 +314,7 @@ This rested on `Locale` and `Policy` keeping their 2.x shape after the facade's A chosen tag selects its own file and its ancestors', never its descendants': `pt-BR` selects `pt-BR.toml` and `pt.toml`, and `pt` selects only `pt.toml`, so a caller in Portugal does not turn on the Brazilian agnomens (#843 left the single-letter `F` and `N` to a `pt-BR` pack). No macrolanguage table is read: Malay and Indonesian are separate tags, though BCP 47 counts Indonesian within the Malay macrolanguage, and a caller with names in both chooses both. The Czech, Slovak, Slovenian and Croatian files each hold the words they share, so each language is a pack of its own. Script subtags need no matching rule yet, since no file carries one. -The TOML names `prefer` and `strict`, reserved for phase 3 since phase 1, stay refused, with a message saying that a strength is the caller's choice and never data. +The TOML names `prefer` and `strict`, reserved for phase 3 since phase 1, stay refused, with a message saying that a strength is the caller's choice and never data (the message landed with 3j, 2026-10-31). ### Measuring a pack diff --git a/docs/usage.rst b/docs/usage.rst index cc9edabbb..be929e146 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1451,8 +1451,9 @@ form to pipe somewhere: {"title": "", "given": "John", "middle": "", "family": "Doe", "suffix": "", "nickname": "", "maiden": ""} Add ``--locale`` to parse with a locale pack (for example ``--locale -ru``); see :doc:`locales`. A code no pack has that is a language with -a vocabulary file (``--locale ko``) chooses that language alone. +ru``); see :doc:`locales`. A code no pack has that is a language whose +vocabulary file holds readings (``--locale ko``) chooses that language +alone; one whose file holds none yet (``vi``) needs ``--strict``. ``--strict`` reads only that language's words (see :ref:`strict-choice`), and needs ``--locale``: diff --git a/nameparser/__main__.py b/nameparser/__main__.py index 9389ae426..22026e1ec 100644 --- a/nameparser/__main__.py +++ b/nameparser/__main__.py @@ -56,7 +56,8 @@ def main(argv: list[str] | None = None) -> int: "nameparser.locales. 'ja' divides unspaced names " "when nameparser[ja] is installed. A code no " "pack has but a vocabulary file does (e.g. 'ko') " - "chooses that language alone") + "chooses that language alone; a file holding no " + "reading yet (e.g. 'vi') needs --strict") ap.add_argument("--strict", action="store_true", help="read only the --locale language's vocabulary " "file and native-script words, dropping every " diff --git a/nameparser/_parser.py b/nameparser/_parser.py index 01e98ff04..c644f8e10 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -380,7 +380,8 @@ def parser_for(*locales: Locale, base: Parser | None = None, reading yet (``vi``), which a plain one refuses as changing nothing, and raises ValueError when the packs choose no language, since it would then strip every Latin-script reading - (decisions/vocabulary-model, 2026-10-30 #753, file-strict). + (decisions/vocabulary-model, 2026-10-30 #753, file-strict, and + 2026-10-31 #753, phase 3j). Validation errors raised while applying a pack are wrapped with that pack's identity (rule D2) -- diff --git a/nameparser/vocab/README.md b/nameparser/vocab/README.md index dfdc8ad86..a984145d0 100644 --- a/nameparser/vocab/README.md +++ b/nameparser/vocab/README.md @@ -283,13 +283,13 @@ These names will mean something in a later phase. Writing one now as a key, in a | Name | Phase | |---|---| -| `prefer` | 3 | -| `strict` | 3 | | `bare_only` | 4 | | `leading_only` | 4 | | `trailing_only` | 4 | | `separators` | 4 | +`prefer` and `strict`, reserved for phase 3 by phase 1, are refused for good: phase 3 made each the caller's (a choice of languages prefers its readings, and `parser_for(..., strict=True)` is strict), so the build says a strength is never data. + Phase 2 gave `default` and `attaches` their meaning (2026-10-25, see [Properties](#properties)). The gate's slots carry position and shape already settles a bare letter ([rules/P7](../../docs/design/rules/P7-one-letter-particle-against-the-initial-it-spells.md), a one-letter particle against the initial it spells), so `leading_only`, `trailing_only` and `bare_only` wait for phase 4's shapes, as `separators` waits for its glued words. Two property values are refused for the same reason, with a message saying a later phase: `addresses = "full"` and `"any"`, and `attached` with a `side` other than `"after"`. diff --git a/tests/v2/test_vocab.py b/tests/v2/test_vocab.py index aaf4687ee..75fcc9230 100644 --- a/tests/v2/test_vocab.py +++ b/tests/v2/test_vocab.py @@ -265,7 +265,11 @@ def _w(body: str) -> str: ("no-schema", {"und.toml": '[[word]]\ntext = "jr"\ngeneration = {}\n'}, "needs `schema = 1`"), ("newer-schema", {"und.toml": OK.replace("schema = 1", "schema = 2")}, "newer than this compiler"), ("unknown-top-level-key", {"und.toml": "language = 'en'\n" + OK}, "unknown key 'language'"), - ("reserved-top-level-key", {"und.toml": "prefer = ['pt']\n" + OK}, "reserved for phase 3"), + ("reserved-top-level-key", {"und.toml": "separators = []\n" + OK}, "reserved for phase 4"), + # phase 3 gave the strengths to the caller, so they stay refused as + # data, the message naming the call (3j) + ("prefer-is-not-data", {"und.toml": "prefer = ['pt']\n" + OK}, "'prefer' is not data; a choice"), + ("strict-is-not-data", {"und.toml": _w('[[word]]\ntext = "mba"\ncredential = { strict = true }')}, "'strict' is not data; strictness is the caller's: parser_for"), ("file-name-not-a-tag", {"english.toml": OK}, "BCP 47 tag"), ("file-name-not-canonical-case", {"pt-br.toml": OK}, "BCP 47 tag"), ("file-with-no-entries", {"en.toml": OK, "und.toml": "schema = 1\n"}, "no \\[\\[word\\]\\] entries"), diff --git a/tools/vocab/build.py b/tools/vocab/build.py index 4b3572c88..2e0bbfd70 100644 --- a/tools/vocab/build.py +++ b/tools/vocab/build.py @@ -272,9 +272,17 @@ def _alnum(text: str) -> str: #: position and bare-letter names wait with `separators` for phase 4's #: shapes and glued words. RESERVED = { - "prefer": 3, "strict": 3, "bare_only": 4, "leading_only": 4, "trailing_only": 4, "separators": 4, } +#: Names phase 1 reserved for phase 3 that phase 3 gave to the caller +#: instead: a choice prefers (3b) and `parser_for(..., strict=True)` is +#: strict (3j), so a strength is never data and stays refused, saying so +#: (the RFC's phase 3 API). +NOT_DATA = { + "prefer": "a choice of languages prefers its readings: choose the " + "language with Locale(languages=...)", + "strict": "strictness is the caller's: parser_for(..., strict=True)", +} _TAG = re.compile(r"[a-z]{2,3}(-[A-Z][a-z]{3})?(-([A-Z]{2}|[0-9]{3}))?") #: A file's `[rules]` table (#754, sub-phase 4f): each key a Policy @@ -385,6 +393,8 @@ def _reserved_or_unknown(where: str, key: str, allowed: Iterable[str]) -> VocabE if key in RESERVED: return VocabError(f"{where}: {key!r} is reserved for phase {RESERVED[key]}; " f"it has no behavior yet") + if key in NOT_DATA: + return VocabError(f"{where}: {key!r} is not data; {NOT_DATA[key]}") return VocabError(f"{where}: unknown key {key!r}; allowed: {sorted(allowed)}") From 54a88fe94fbe6dedd5a5191cfcb00b30fc86cade Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 31 Oct 2026 18:05:58 -0700 Subject: [PATCH 4/7] Test the strict CLI errors in process, and the en fallback of the strict refusal (#753, 3j) Codecov counted the CLI's new error branches uncovered: their tests ran the command in a subprocess, which coverage does not see. They now call main() in process; a TR_AZ row covers the refusal offering en for a pack whose code is no language. Co-Authored-By: Claude Opus 5.5 --- tests/v2/test_cli.py | 50 ++++++++++++++++++++++++++--------------- tests/v2/test_parser.py | 2 ++ 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/tests/v2/test_cli.py b/tests/v2/test_cli.py index afa776b01..b6ac76519 100644 --- a/tests/v2/test_cli.py +++ b/tests/v2/test_cli.py @@ -275,28 +275,42 @@ def test_cli_locale_falls_back_to_a_language_and_strict_reads_it_alone( assert _roles(_fields(capsys, *argv)) == roles -def test_cli_strict_without_a_locale_is_a_usage_error() -> None: - proc = _run("Kim Do Hyun", "--strict") - assert proc.returncode == 2 - assert "--strict needs --locale" in proc.stderr +def _usage_error(capsys: pytest.CaptureFixture[str], *argv: str) -> str: + # in process, so coverage sees the branch: argparse exits 2 + with pytest.raises(SystemExit) as info: + cli.main([*argv, "John Smith"]) + assert info.value.code == 2 + return capsys.readouterr().err -def test_cli_locale_neither_a_pack_nor_a_language_lists_both() -> None: - proc = _run("John Smith", "--locale", "xx", "--strict") - assert proc.returncode == 2 +def test_cli_strict_without_a_locale_is_a_usage_error( + capsys: pytest.CaptureFixture[str]) -> None: + assert "--strict needs --locale" in _usage_error(capsys, "--strict") + + +def test_cli_locale_neither_a_pack_nor_a_language_lists_both( + capsys: pytest.CaptureFixture[str]) -> None: + err = _usage_error(capsys, "--locale", "xx", "--strict") # the pack codes, then the language tags - assert "tr_az" in proc.stderr and "nor a language" in proc.stderr - assert "ko" in proc.stderr.split("nor a language")[1] + assert "tr_az" in err and "nor a language" in err + assert "ko" in err.split("nor a language")[1] -def test_cli_a_language_with_no_reading_needs_strict() -> None: - proc = _run("Do Van Minh", "--locale", "vi") - assert proc.returncode == 2 - assert "vi.toml holds no reading yet" in proc.stderr - assert "(--strict) accepts it" in proc.stderr +def test_cli_locale_that_is_no_tag_at_all_lists_the_packs( + capsys: pytest.CaptureFixture[str]) -> None: + # "" cannot even be a tag, so the stand-in Locale refuses it + err = _usage_error(capsys, "--locale", "") + assert "tr_az" in err and "nor a language" not in err -def test_cli_strict_with_a_pack_whose_code_is_no_language() -> None: - proc = _run("John Smith", "--locale", "tr_az", "--strict") - assert proc.returncode == 2 - assert "chooses no language, and its code is not one" in proc.stderr +def test_cli_a_language_with_no_reading_needs_strict( + capsys: pytest.CaptureFixture[str]) -> None: + err = _usage_error(capsys, "--locale", "vi") + assert "vi.toml holds no reading yet" in err + assert "(--strict) accepts it" in err + + +def test_cli_strict_with_a_pack_whose_code_is_no_language( + capsys: pytest.CaptureFixture[str]) -> None: + err = _usage_error(capsys, "--locale", "tr_az", "--strict") + assert "chooses no language, and its code is not one" in err diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 9726b342c..7b139071e 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -2650,6 +2650,8 @@ def test_a_strict_choice_drops_mul_and_keeps_native_script_vocabulary() -> None: ((), "no pack was given"), (("ZH",), "the pack 'zh' chooses none"), (("ZH", "RU"), "the packs 'zh', 'ru' choose none"), + # a code that is no language: the fix offered chooses en + (("TR_AZ",), "the pack 'tr_az' chooses none.*'en'"), ]) def test_a_strict_call_choosing_no_language_is_refused_with_a_fix_that_runs( packs: tuple[str, ...], match: str) -> None: From 6975146581ed3ee1b9a57db5631572690ca58a6e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 31 Oct 2026 18:08:19 -0700 Subject: [PATCH 5/7] Review fixes: a pack's region names the pack, tr_az --strict names a tag, supersession marks (#753, 3j) --locale zh-CN, es-MX and ja-JP had fallen back to a bare choice, dropping the pack's segmentation, policy, list and segmenter in silence; each now exits 2 naming the pack, while pt-BR resolves to pt_br. --locale tr_az --strict now names a tag to give instead. The 2026-10-30 file-strict entry's head marks every passage 3j decided or reversed, and the 3j entry points at docs/locales.rst, where the bare spelling is taught. A mixed-script row holds is_native_script to 'no Latin letter'. Co-Authored-By: Claude Opus 5.5 --- docs/design/decisions/vocabulary-model.md | 8 ++++---- docs/usage.rst | 3 ++- nameparser/__main__.py | 21 +++++++++++++++++---- tests/v2/test_cli.py | 22 +++++++++++++++++++++- tests/v2/test_reading_index.py | 2 ++ 5 files changed, 46 insertions(+), 10 deletions(-) diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index 3f7b900b7..b3df2f61d 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -414,7 +414,7 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) - **Recompute:** `uv run pytest -n0 tests/v2/test_shape_inventory.py`; the counts by order from `ARRANGEMENTS` and `SHAPES` (each row's order is `SHAPES[row.shape].order`); the corpus and snapshot counts from `git diff --numstat` against `1f6bdf1d` after `uv run python tools/differential/build_shapes_corpus.py` and `uv run python tools/differential/snapshot.py --record`; the control by parsing each family-first row with `Parser()`. - **Disclosed, not added:** the two title-particles at the head written with periods (`Freiherr St. van Berg MA`, `Freiherr. St. van Berg MA`) read as the bare spelling does, the title run whole, under every order. - **Open, unweighed and held by no issue, so these rows carry no mark:** an ambiguous particle opening a family-first name is left where position puts it and reported, as root AGENTS.md's particles paragraph describes, though no decision weighs it there (`van Gogh Vincent` under `FAMILY_FIRST`: family `van`, given `Gogh`; `du Ménil Thi Mai`, its letter decomposed, under `FAMILY_FIRST_GIVEN_LAST`: family `du`, middle `Ménil Thi`); rules/P1 covers a never-given particle only. And `Vega y Lopez Jr., PhD Jones` under the family-first orders reads given `y`, middle `Lopez` (`FAMILY_FIRST`) and given `Lopez`, middle `y` (`FAMILY_FIRST_GIVEN_LAST`): the default order's division of the words laid out by the order, which decisions/C3 records under the default order only. And a Hungarian title after a married `-né` word, before the woman's birth family name (`Bakainé dr. Korbács Melinda`), 102 of 37,250 names in the 2024 municipal candidates and every one misread today (decisions/H5's #938 entry has the measurement): what licenses the title there is the `-né` word, so it is a question for a future `hu` pack, which no sub-phase schedules, and not for a general rule. -- **2026-10-30 #753, file-strict (its proposed `locales.choose` and its bullet that strict `vi` needs vi.toml readings first superseded by 2026-10-31 #753, phase 3j) — STRICT IS FILE-STRICT: A STRICT CHOICE READS ONLY THE CHOSEN LANGUAGES' FILES AND NATIVE-SCRIPT VOCABULARY, NOT `mul.toml` OR `und.toml`, SO IT NEEDS NO 3i; THE `strict=True` ARGUMENT STANDS, THE REST OF ITS API IS PROPOSED, NOT DECIDED, AND NOTHING IS BUILT.** SUPERSEDES: decision 6's strict variant in the 2026-10-15/16 entry, and in the 2026-10-27 #753 entry item 2's definition of strict (other languages' name readings and their markers leave), item 4 (strict waits until name readings say whose names they are) and 3i in item 11. Item 1's `strict=True` argument of `parser_for`, over the whole choice, stands. The definition and items 1 to 3 were decided in chat on 2026-10-30; the measurements were taken on 2026-10-31 on master `b577e9a8`. The design overview is docs/design/vocabulary-model-rfc.md's "Strict: the chosen files and nothing else". +- **2026-10-30 #753, file-strict (its verdict's "the rest of its API is proposed, not decided, and nothing is built", its "Proposed, not decided" bullet, decided save `locales.choose`, which is reversed, and its files-with-no-readings bullet's clause that strict `vi` needs vi.toml readings first superseded by 2026-10-31 #753, phase 3j) — STRICT IS FILE-STRICT: A STRICT CHOICE READS ONLY THE CHOSEN LANGUAGES' FILES AND NATIVE-SCRIPT VOCABULARY, NOT `mul.toml` OR `und.toml`, SO IT NEEDS NO 3i; THE `strict=True` ARGUMENT STANDS, THE REST OF ITS API IS PROPOSED, NOT DECIDED, AND NOTHING IS BUILT.** SUPERSEDES: decision 6's strict variant in the 2026-10-15/16 entry, and in the 2026-10-27 #753 entry item 2's definition of strict (other languages' name readings and their markers leave), item 4 (strict waits until name readings say whose names they are) and 3i in item 11. Item 1's `strict=True` argument of `parser_for`, over the whole choice, stands. The definition and items 1 to 3 were decided in chat on 2026-10-30; the measurements were taken on 2026-10-31 on master `b577e9a8`. The design overview is docs/design/vocabulary-model-rfc.md's "Strict: the chosen files and nothing else". 1. **The definition.** A strict choice compiles only the chosen languages' files, each tag selecting its own and its ancestors', plus native-script vocabulary, which stays on because it selects itself by script (the CJK honorifics; the Korean surname list, outside the per-word model, is in no choice's compile and stays too); everything else leaves, vocabulary and name readings alike. A word with no reading in what is compiled is read by position and shape alone, so under strict `vi`, `do`, which only en.toml and pt.toml hold, is a name word, while the shape rules still act (`Dr. Minh Do` keeps title `Dr.`, rules/H2; `Minh Do, M.D., Ph.D.` keeps suffix `Ph.D.` past the second comma, rules/C2, but reads given `M.D.`). For a credential, the harm item 4 of the 2026-10-27 entry guarded against (`Minh Do` → suffix `Do`) does not arise today, measured 2026-10-31: no strict choice of one tag ungates a credential acronym, each credential's gating name reading sitting in its own file. It holds only while name readings are added to more files and never moved: moving `do`'s Đỗ readings from en.toml to vi.toml would ungate its credential under strict `en`. For the other kinds it can, where the vocabulary reading is in a chosen file and the gating name reading only in a dropped one (the ungated-words bullet below; strict `pt` reads `Do Van Minh` family `Do Van Minh`). 3i, the attribute naming each name reading's bearers' languages, is still not needed for strict: under file-strict that case is remedied by data, a name reading written in the chosen file whose names carry the word. Name-strict, the earlier definition, under which other languages' titles, credentials and particles stayed and only their name readings left, is set aside with 3i. 2. **`mul.toml` and `und.toml` are not in a strict choice.** `mul.toml` stays as it is: its entries name several languages each or "the European languages" (`dr`, `prof`), or carry no note (the numerals, `&`), so under file-strict it behaves as one more language's file. Open: entries may later move between en.toml and mul.toml, or into the files of the languages that write them, so that strict `en` can shed problem words; which words those are is not yet known. 3. **Dropping English credentials under strict `vi` is accepted in principle.** Vietnamese writes its degrees in Vietnamese, before the name (`TS.`, `ThS.`, `GS.`, `PGS.`, `BS.`, stacked as `PGS.TS.`), and has no generational suffix; English `PhD` and `MD` come with English-context writing. Measured below rather than assumed. @@ -447,14 +447,14 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) - **3i's evidence, kept** from the RFC paragraph this entry retires, should 3i be wanted again: all measured 2026-10-28 on master `c8357e74`, not on this entry's tree: of the 227 name readings, 165 words carried one; `do`'s family box led by Brazil in ORCID (none of Brazil's 7,227 `do` alone, 2,629 opening with it and 4,598 holding it inside) but by share Vietnam (1.90% against 0.81%; PatentsView 1.78% and 0.64%), Vietnam having 618 boxes of `do` alone and Korea 151; `kim`'s family box 17.67% of ORCID's Korean holders and 13.30% of PatentsView's, its given box 0.93% of PatentsView's Danish inventors. Recompute: `uv run python tools/name_usage.py do kim --no-wikidata --boxes`, and the whole-box split from the cached ORCID extract's `family` and `country` keys. - **Candidate next steps, not taken here:** Vietnamese degree prefixes as vi.toml readings, each checked for collisions and bearers per nameparser/vocab/AGENTS.md first; `ông` and `bà` stay excluded, Ông being a Vietnamese surname, a clash within Vietnamese strict does not settle; 3h's `nl` prepositions and 3e's `sushri` as data with the clashing name readings declared in ko.toml, vi.toml and or.toml, gated under the plain choice and not under strict, though the plain `nl` choice still reads `Park In Hwan` family `In Hwan` and `Nguyen To Nga` family `To Nga`, item 4's recorded cost, the marker acting at the head only (re-run 2026-10-31), which a follow-up must settle before shipping them; `_reading_index`'s `UNCHOOSABLE` comment and `selected`'s refusal text, which say no choice removes a reading of `mul` or `und`, updated with 3j; the 16 ungated words read one by one. - **Recompute:** a strict choice of the tags `T` is `compile_readings(choose(rows, T))` from `nameparser._reading_index`, `rows` the `index()` rows whose tag is in `T` or whose text has a letter and no letter whose `unicodedata.name` contains `LATIN`, applied to `Lexicon.default()` as `_parser._apply_choice` applies `_choice_delta` (the delta taken against `compile_choice(frozenset())`, the markers of a dropped base entry removed), and a pack's own patch and fragment applied by `parser_for(dataclasses.replace(pack, languages=frozenset()), base=Parser(lexicon=that))`. Count a word's share of the moves as the moves whose name, split at spaces and commas with edge periods dropped and casefolded, holds it. Score ORCID with `local_agreement.read(record, local_agreement.FIELDED["orcid-researchers"], parser, local_agreement._title_words(parser.lexicon))` over the cache's `raw.jsonl` records of each country, and PSC the same with `"psc-individuals"` and the `nationality` key; the title-word filter is the given box's first word, split at spaces and periods and casefolded, in {dr, prof, professor, mr, mrs, ms, miss, dra, ing, mgr, assoc, assist, er, engr, drs, ir}. The ungated words: for each choosable tag, the words in a marker (`particles_ambiguous`, `suffix_acronyms_ambiguous`, `suffix_words_ambiguous`, `titles_ambiguous`) of the plain choice of that tag (`parser_for(Locale(code="x", lexicon=Lexicon.empty(), languages={tag})).lexicon`) that stay in the base set of the strict lexicon and leave the marker, or join `titles`; compared with `Lexicon.default()`'s markers instead, it misses the readings a choice turns on (`st` under cs, hr, sk and sl, `nic` under gd) and finds 15 words. The scripts are in no tree: they are a few dozen lines each from this recipe. -- **2026-10-31 #753, phase 3j — FILE-STRICT IS BUILT: `parser_for(..., strict=True)` COMPILES ONLY THE CHOSEN FILES AND NATIVE-SCRIPT VOCABULARY; A STRICT CALL CHOOSING NO LANGUAGE RAISES; NO `locales.choose`; THE WORDS STRICT UNGATES ARE DERIVED AND HELD BY A TEST; A FILE HOLDING NO READING IS A STRICT CHOICE ONLY; THE DEFAULT DOES NOT MOVE.** SUPERSEDES: in the 2026-10-30 #753, file-strict entry, the proposed `locales.choose(*tags)` and the files-with-no-readings bullet's "strict `vi` needs vi.toml readings first". Carries out 3j of item 11 of the 2026-10-27 #753 entry, with the definition of the 2026-10-30 entry. The four API points below were decided on 2026-10-31; the rest of the 2026-10-30 entry's proposed bullet is built as proposed (no `Locale.strict` field; the chosen files' `default = false` readings turned on and no other file's; patches and fragments applied after the strict compile, untouched; `Lexicon.given_names` and `Lexicon.surnames` outside it; packs stacked as a union; a caller's listing of a dropped word dropped by the delta). Built on master `8438ccbb`. +- **2026-10-31 #753, phase 3j — FILE-STRICT IS BUILT: `parser_for(..., strict=True)` COMPILES ONLY THE CHOSEN FILES AND NATIVE-SCRIPT VOCABULARY; A STRICT CALL CHOOSING NO LANGUAGE RAISES; NO `locales.choose`; THE WORDS STRICT UNGATES ARE DERIVED AND HELD BY A TEST; A FILE HOLDING NO READING IS A STRICT CHOICE ONLY; THE DEFAULT DOES NOT MOVE.** SUPERSEDES: in the 2026-10-30 #753, file-strict entry, its verdict's "the rest of its API is proposed, not decided, and nothing is built"; its "Proposed, not decided" bullet, every item decided as proposed save `locales.choose(*tags)`, which is left out; and the files-with-no-readings bullet's "strict `vi` needs vi.toml readings first". Carries out 3j of item 11 of the 2026-10-27 #753 entry, with the definition of the 2026-10-30 entry. The four API points below were decided on 2026-10-31; the rest of the 2026-10-30 entry's proposed bullet is built as proposed (no `Locale.strict` field; the chosen files' `default = false` readings turned on and no other file's; patches and fragments applied after the strict compile, untouched; `Lexicon.given_names` and `Lexicon.surnames` outside it; packs stacked as a union; a caller's listing of a dropped word dropped by the delta). Built on master `8438ccbb`. 1. **A strict call that chooses no language raises `ValueError`.** The message names the packs given and offers a pack that chooses a language, the first pack's code where a strict choice accepts it as one (`parser_for(locales.ZH, strict=True)` offers `Locale(code='zh', lexicon=Lexicon.empty(), languages=frozenset({'zh'}))`), else `en`; tests/v2/test_parser.py runs the offered code. - 2. **No public `locales.choose`.** A bare choice is a `Locale` with an empty lexicon and the tags, the spelling docs/usage.rst teaches. + 2. **No public `locales.choose`.** A bare choice is a `Locale` with an empty lexicon and the tags, the spelling docs/locales.rst teaches. 3. **The words strict ungates are derived**, `_reading_index.strict_ungated`, each strict choice against the plain choice of the same tags and never against the default's markers, which miss a reading the choice turns on; tests/v2/test_reading_index.py holds every one-tag strict choice to a per-tag table, which is the 2026-10-30 entry's 16 words and 27 word-and-tag pairs over 13 tags exactly, so a word that joins or leaves is a reviewed diff. 4. **A strict choice accepts a tag whose file holds no reading** (hu, kk, kok, vi and zu today): under strict it is not inert, every Latin-script reading leaving. A plain choice of one is still refused, its message now saying a strict one accepts it. The reading index records every file's tag for this (`FILES` in nameparser/config/_index.py, read by `_reading_index.files`), a file holding only exclusions writing no row of `READINGS`. - The build refuses `prefer` and `strict` as data, saying a strength is the caller's (`parser_for(..., strict=True)`), where it had said each was reserved for phase 3; the RFC's phase 3 API section had decided the message. - `_reading_index`'s `UNCHOOSABLE` comment and `selected`'s refusal text, which said no choice removes a reading of `mul` or `und`, now say a plain choice never does and a strict one drops both. - - **The command line** gains `--strict`, which needs `--locale` (exit 2 without it). `--locale` falls back from a pack code to a bare choice of a language tag (`--locale ko`), and a code that is neither exits 2 listing the pack codes and the tags. Under `--strict`, a pack that chooses no language chooses its own code (`--locale zh --strict`), and one whose code is no language (`tr_az`) exits 2. These are the debug helper's, chosen with the build rather than decided as API. + - **The command line** gains `--strict`, which needs `--locale` (exit 2 without it). `--locale` falls back from a pack code to a bare choice of a language tag (`--locale ko`), and a code that is neither exits 2 listing the pack codes and the tags. Under `--strict`, a pack that chooses no language chooses its own code (`--locale zh --strict`), and one whose code is no language (`tr_az`) exits 2, naming a tag to give instead. A region of a pack's language (`zh-CN`, `es-MX`, `ja-JP`) exits 2 naming the pack, since a bare choice of it would drop the pack's segmentation, policy and segmenter in silence (the RFC's reason for not folding a bare tag into `locales.get`), while a tag spelled for a pack's code (`pt-BR`) is that pack. These are the debug helper's, chosen with the build rather than decided as API. - **Measured: the 2026-10-30 entry's ORCID table re-run on the built code**, its recipe with `parser_for(pack, strict=True)` (and a bare `en` pack beside it for strict + en) in place of the simulation, on this phase's branch over `8438ccbb`, Python 3.11: every cell matches, 14 countries by four columns. - **Measured: the worked examples on the built code**, default order, against the default: under strict `ko`, `Kim Do Hyun` reads middle `Do`, family `Hyun` (default family `Do Hyun`, pt's particle), `Kim Bin Hyun` middle `Bin` (ar's and ms's particle), `Ra Min Jun` given `Ra` (de's academic title), `Kim Min Ra` family `Ra` (en's credential), back to suffix `Ra` under strict `ko` + `en`; strict `ja` reads `Gen Hoshino` given `Gen` (default title), strict `zh` `De Xu` given `De`, family `Xu` (default family `De Xu`). Every example the 2026-10-30 entry gives parses on the built code as it says. - The default does not move: tests/v2/test_snapshot.py passes unchanged, a plain choice compiling as before. diff --git a/docs/usage.rst b/docs/usage.rst index be929e146..cf7df97df 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1453,7 +1453,8 @@ form to pipe somewhere: Add ``--locale`` to parse with a locale pack (for example ``--locale ru``); see :doc:`locales`. A code no pack has that is a language whose vocabulary file holds readings (``--locale ko``) chooses that language -alone; one whose file holds none yet (``vi``) needs ``--strict``. +alone; one whose file holds none yet (``vi``) needs ``--strict``, and +a region of a pack's language (``zh-CN``) exits naming the pack to use. ``--strict`` reads only that language's words (see :ref:`strict-choice`), and needs ``--locale``: diff --git a/nameparser/__main__.py b/nameparser/__main__.py index 22026e1ec..598a9d039 100644 --- a/nameparser/__main__.py +++ b/nameparser/__main__.py @@ -80,13 +80,24 @@ def main(argv: list[str] | None = None) -> int: # exit 2 listing the codes, not silently fall back to the default if args.locale is not None: from nameparser import locales + # a tag spelled for a pack's code is that pack (pt-BR is pt_br); + # a region of a pack's language (zh-CN, es-MX, ja-JP) is refused + # naming the pack, never chosen bare: a bare choice would drop + # the pack's list, policy and segmenter in silence + code = args.locale.replace("-", "_").casefold() + language = code.split("_")[0] try: - pack = locales.get(args.locale) + pack = locales.get(code if code in locales.available() + else args.locale) except KeyError as exc: + unknown = exc.args[0] + if language != code and language in locales.available(): + ap.error(f"{unknown}; {args.locale!r} is a region of " + f"the language of the pack {language!r}: use " + f"--locale {language}") # not a pack: a language tag chooses that language alone, # which parser_for checks below; if it is neither, the error # lists the codes and the tags - unknown = exc.args[0] try: pack = Locale(code="cli_language", lexicon=Lexicon.empty(), languages=frozenset({args.locale})) @@ -99,8 +110,10 @@ def main(argv: list[str] | None = None) -> int: pack = dataclasses.replace( pack, languages=frozenset({pack.code})) except ValueError: - ap.error(f"--strict: the pack {pack.code!r} chooses no " - f"language, and its code is not one") + ap.error(f"--strict needs a language to keep: the pack " + f"{pack.code!r} chooses none, and its code is not " + f"a language tag; give a tag instead (e.g. " + f"--locale {pack.code.split('_')[0]} --strict)") # The pack only activates division; the installed extra divides. # Without it a note says so in the command line's terms, in # place of the parser's warning (rules/D1), whose other remedies diff --git a/tests/v2/test_cli.py b/tests/v2/test_cli.py index b6ac76519..74e39f448 100644 --- a/tests/v2/test_cli.py +++ b/tests/v2/test_cli.py @@ -313,4 +313,24 @@ def test_cli_a_language_with_no_reading_needs_strict( def test_cli_strict_with_a_pack_whose_code_is_no_language( capsys: pytest.CaptureFixture[str]) -> None: err = _usage_error(capsys, "--locale", "tr_az", "--strict") - assert "chooses no language, and its code is not one" in err + assert "--strict needs a language" in err + assert "--locale tr --strict" in err + + +@pytest.mark.parametrize("tag,pack", [ + # a bare choice would drop the pack's segmentation, two family + # names and given-name list, and the segmenter, in silence + ("zh-CN", "zh"), ("es-MX", "es"), ("ja-JP", "ja"), ("pt-PT", "pt"), +]) +def test_cli_a_region_of_a_packs_language_names_the_pack( + capsys: pytest.CaptureFixture[str], tag: str, pack: str) -> None: + err = _usage_error(capsys, "--locale", tag) + assert f"use --locale {pack}" in err + + +@pytest.mark.parametrize("tag", ["pt-BR", "pt_br", "PT-BR"]) +def test_cli_a_tag_spelled_for_a_packs_code_is_that_pack( + capsys: pytest.CaptureFixture[str], tag: str) -> None: + assert (_fields(capsys, "--locale", tag, "Pedro Silva Sb") + == _fields(capsys, "--locale", "pt_br", "Pedro Silva Sb")) + assert _fields(capsys, "--locale", tag, "Pedro Silva Sb")["suffix"] == "Sb" diff --git a/tests/v2/test_reading_index.py b/tests/v2/test_reading_index.py index 629489386..add5e098d 100644 --- a/tests/v2/test_reading_index.py +++ b/tests/v2/test_reading_index.py @@ -301,6 +301,8 @@ def test_holds_kind_reads_every_file_whatever_the_default( @pytest.mark.parametrize("text,native", [ ("씨", True), ("господин", True), ("श्री", True), ("殿", True), ("dr", False), ("ông", False), ("&", False), ("2", False), + # one Latin letter makes a word Latin, however many others it holds + ("dr김", False), ("김x", False), ]) def test_native_script_is_a_letter_and_no_latin_letter( text: str, native: bool) -> None: From e40c97fbc9f47a76f2b5058ba009370afbdcb609 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 31 Oct 2026 18:12:26 -0700 Subject: [PATCH 6/7] Decisions: the CLI region refusal's own reason, and sushri no longer waits for strict (#753, 3j) Co-Authored-By: Claude Opus 5.5 --- docs/design/decisions/vocabulary-model.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index b3df2f61d..d4aeb7292 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -12,7 +12,7 @@ The vocabulary was a set of base sets plus marker subsets (`particles` and `part - A patronymic field is to come in 3.0, as a view that names its words without emptying `family`, and the particle kinds already draw the `clan`/`patronymic` line for it: a patronymic is followed by a parent's name, a clan prefix by a family name the lineage shares — 2026-10-15/16; 2026-10-19 #749; 2026-10-28 #754 (Derek); landed as rules/R9, 2026-10-29 #754, sub-phase 4c. - A shipped pack that is only a choice of languages declares no `DEVIATES`, its surface the words its choice changes; `ga` turns on `nic`, `mhic`, `mic` and `uí` as clan particles, `gd` `nic`, and `pt-BR` `sb` as Sobrinho's spelling, while the single letters `f` and `n` are excluded, the pack included, the letter abbreviating a surname more often than the agnomen in the filings that write it — 2026-10-30 #753, phase 3h. - `es` turns on the courtesy, academic and professional titles that lead a Spanish name with no complement (`lic`, `licenciado`, `licenciada`, `lcdo`, `lcda`, `ingeniero`, `ingeniera`, `arq`, `mtro`, `mtra`, `doctora`, `profesor`, `profesora`, `doña`) and `pt` `doutor`, `doutora`, `professora` and `profa`, `pt-BR` with it; `don`, `dona` and `dom` are excluded, the packs included, names opening with each as a name more often than as the title; `maestro`, a family name whose form of address is written `Mtro.`, is excluded too, and so is `h`, the hijo an initial is written as; `sr` keeps a generation reading beside its salutation in both files, so a choice leaves it the dual position decides; no connective continues a title run, the shape unwritten in name data, and the office titles that take a complement stay out — 2026-10-30 #753, phase 3g. -- Of the romanized titles a pack's phase lists, one whose native spelling is a default title turns on unless, counted in persons and pooled across every population measured, names open with it more often than it opens one as the title; a spelling outside the phase's list (`Shreemati`, `Srimati`) stays open until a phase writes it as a reading: `hi` turns on `smt`, `shrimati` and `shriyut`, `bn` `smt`, `shrimati` and `adhyapak` (`shriyut` with no bearer anywhere, `adhyapak` with family-name bearers only, every one where the title claim, acting at the head, does not reach), while `shri`, opening given names in every population measured, stays excluded, and `sushri`, an Odia given name, waits for strict with no reading written, a plain choice being made for data that holds those names — 2026-10-30 #753, phase 3e. +- Of the romanized titles a pack's phase lists, one whose native spelling is a default title turns on unless, counted in persons and pooled across every population measured, names open with it more often than it opens one as the title; a spelling outside the phase's list (`Shreemati`, `Srimati`) stays open until a phase writes it as a reading: `hi` turns on `smt`, `shrimati` and `shriyut`, `bn` `smt`, `shrimati` and `adhyapak` (`shriyut` with no bearer anywhere, `adhyapak` with family-name bearers only, every one where the title claim, acting at the head, does not reach), while `shri`, opening given names in every population measured, stays excluded, and `sushri`, an Odia given name, waits with no reading written for data strict can read since 3j, a `default = false` salutation in hi.toml with the Odia name declared in or.toml (2026-10-31 #753, phase 3j), a plain choice being made for data that holds those names — 2026-10-30 #753, phase 3e. - A language pack is a choice of languages on `Locale.languages` selecting which readings compile, a tag choosing its own file and its ancestors', and a reading index, loaded on its first lookup, carries the languages to run time; strict is file-strict, the chosen files and native-script vocabulary only, `mul.toml` and `und.toml` not among them, so it needs no 3i; third-party language files are deferred — 2026-10-27 #753 (Derek); 2026-10-29 #753, phase 3a (Derek); 2026-10-29 #753, phase 3b; 2026-10-30 #753, file-strict. - A strict choice is `parser_for(..., strict=True)` over the whole choice, applied as a delta as a plain choice is; a strict call whose packs choose no language raises; a strict choice accepts a tag whose file holds no reading, which a plain one refuses as changing nothing; a bare choice has no constructor, no `locales.choose`; and the words strict ungates are derived against the plain choice of the same tags and held to a reviewed set by a test — 2026-10-31 #753, phase 3j. - A choice is matched without regard to case, a tag whose file does not exist selects its ancestors' (`pt-PT` selects `pt`), `mul` and `und` are no language to choose, and a reading turned on also prefers, so `de`'s generation `i` drops `ca`'s and `pl`'s conjunction under a `de` choice; a choice is one call's: one chained through `base=` applies on top of the base's rather than unioning with it, and a choice drops the caller's own base entries it removes — 2026-10-29 #753, phase 3b; 2026-10-29 #753, phase 3b, chained choices (Derek). @@ -454,7 +454,7 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) 4. **A strict choice accepts a tag whose file holds no reading** (hu, kk, kok, vi and zu today): under strict it is not inert, every Latin-script reading leaving. A plain choice of one is still refused, its message now saying a strict one accepts it. The reading index records every file's tag for this (`FILES` in nameparser/config/_index.py, read by `_reading_index.files`), a file holding only exclusions writing no row of `READINGS`. - The build refuses `prefer` and `strict` as data, saying a strength is the caller's (`parser_for(..., strict=True)`), where it had said each was reserved for phase 3; the RFC's phase 3 API section had decided the message. - `_reading_index`'s `UNCHOOSABLE` comment and `selected`'s refusal text, which said no choice removes a reading of `mul` or `und`, now say a plain choice never does and a strict one drops both. - - **The command line** gains `--strict`, which needs `--locale` (exit 2 without it). `--locale` falls back from a pack code to a bare choice of a language tag (`--locale ko`), and a code that is neither exits 2 listing the pack codes and the tags. Under `--strict`, a pack that chooses no language chooses its own code (`--locale zh --strict`), and one whose code is no language (`tr_az`) exits 2, naming a tag to give instead. A region of a pack's language (`zh-CN`, `es-MX`, `ja-JP`) exits 2 naming the pack, since a bare choice of it would drop the pack's segmentation, policy and segmenter in silence (the RFC's reason for not folding a bare tag into `locales.get`), while a tag spelled for a pack's code (`pt-BR`) is that pack. These are the debug helper's, chosen with the build rather than decided as API. + - **The command line** gains `--strict`, which needs `--locale` (exit 2 without it). `--locale` falls back from a pack code to a bare choice of a language tag (`--locale ko`), and a code that is neither exits 2 listing the pack codes and the tags. Under `--strict`, a pack that chooses no language chooses its own code (`--locale zh --strict`), and one whose code is no language (`tr_az`) exits 2, naming a tag to give instead. A region of a pack's language (`zh-CN`, `es-MX`, `ja-JP`) exits 2 naming the pack, since a bare choice of it would drop the pack's segmentation, policy and segmenter in silence (found by review of this PR: `--locale zh-CN 王小明` read family `王小明` where `--locale zh` divides it; the RFC declines folding a bare tag into `locales.get` for a reason of its own, a choice no test has measured), while a tag spelled for a pack's code (`pt-BR`) is that pack. These are the debug helper's, chosen with the build rather than decided as API. - **Measured: the 2026-10-30 entry's ORCID table re-run on the built code**, its recipe with `parser_for(pack, strict=True)` (and a bare `en` pack beside it for strict + en) in place of the simulation, on this phase's branch over `8438ccbb`, Python 3.11: every cell matches, 14 countries by four columns. - **Measured: the worked examples on the built code**, default order, against the default: under strict `ko`, `Kim Do Hyun` reads middle `Do`, family `Hyun` (default family `Do Hyun`, pt's particle), `Kim Bin Hyun` middle `Bin` (ar's and ms's particle), `Ra Min Jun` given `Ra` (de's academic title), `Kim Min Ra` family `Ra` (en's credential), back to suffix `Ra` under strict `ko` + `en`; strict `ja` reads `Gen Hoshino` given `Gen` (default title), strict `zh` `De Xu` given `De`, family `Xu` (default family `De Xu`). Every example the 2026-10-30 entry gives parses on the built code as it says. - The default does not move: tests/v2/test_snapshot.py passes unchanged, a plain choice compiling as before. From 60cd8213efea4171471d9d29cf1c60751e6eec30 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 31 Oct 2026 18:14:26 -0700 Subject: [PATCH 7/7] CLI: a pack's language with a subtag names the pack without calling it a region (#753, 3j) Co-Authored-By: Claude Opus 5.5 --- docs/design/decisions/vocabulary-model.md | 2 +- docs/usage.rst | 3 ++- nameparser/__main__.py | 7 ++++--- tests/v2/test_cli.py | 6 ++++-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index d4aeb7292..40ab5db63 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -454,7 +454,7 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) 4. **A strict choice accepts a tag whose file holds no reading** (hu, kk, kok, vi and zu today): under strict it is not inert, every Latin-script reading leaving. A plain choice of one is still refused, its message now saying a strict one accepts it. The reading index records every file's tag for this (`FILES` in nameparser/config/_index.py, read by `_reading_index.files`), a file holding only exclusions writing no row of `READINGS`. - The build refuses `prefer` and `strict` as data, saying a strength is the caller's (`parser_for(..., strict=True)`), where it had said each was reserved for phase 3; the RFC's phase 3 API section had decided the message. - `_reading_index`'s `UNCHOOSABLE` comment and `selected`'s refusal text, which said no choice removes a reading of `mul` or `und`, now say a plain choice never does and a strict one drops both. - - **The command line** gains `--strict`, which needs `--locale` (exit 2 without it). `--locale` falls back from a pack code to a bare choice of a language tag (`--locale ko`), and a code that is neither exits 2 listing the pack codes and the tags. Under `--strict`, a pack that chooses no language chooses its own code (`--locale zh --strict`), and one whose code is no language (`tr_az`) exits 2, naming a tag to give instead. A region of a pack's language (`zh-CN`, `es-MX`, `ja-JP`) exits 2 naming the pack, since a bare choice of it would drop the pack's segmentation, policy and segmenter in silence (found by review of this PR: `--locale zh-CN 王小明` read family `王小明` where `--locale zh` divides it; the RFC declines folding a bare tag into `locales.get` for a reason of its own, a choice no test has measured), while a tag spelled for a pack's code (`pt-BR`) is that pack. These are the debug helper's, chosen with the build rather than decided as API. + - **The command line** gains `--strict`, which needs `--locale` (exit 2 without it). `--locale` falls back from a pack code to a bare choice of a language tag (`--locale ko`), and a code that is neither exits 2 listing the pack codes and the tags. Under `--strict`, a pack that chooses no language chooses its own code (`--locale zh --strict`), and one whose code is no language (`tr_az`) exits 2, naming a tag to give instead. A pack's language with a region or script subtag (`zh-CN`, `zh-Hant`, `es-MX`, `ja-JP`) exits 2 naming the pack, since a bare choice of it would drop the pack's segmentation, policy and segmenter in silence (found by review of this PR: `--locale zh-CN 王小明` read family `王小明` where `--locale zh` divides it; the RFC declines folding a bare tag into `locales.get` for a reason of its own, a choice no test has measured), while a tag spelled for a pack's code (`pt-BR`) is that pack. These are the debug helper's, chosen with the build rather than decided as API. - **Measured: the 2026-10-30 entry's ORCID table re-run on the built code**, its recipe with `parser_for(pack, strict=True)` (and a bare `en` pack beside it for strict + en) in place of the simulation, on this phase's branch over `8438ccbb`, Python 3.11: every cell matches, 14 countries by four columns. - **Measured: the worked examples on the built code**, default order, against the default: under strict `ko`, `Kim Do Hyun` reads middle `Do`, family `Hyun` (default family `Do Hyun`, pt's particle), `Kim Bin Hyun` middle `Bin` (ar's and ms's particle), `Ra Min Jun` given `Ra` (de's academic title), `Kim Min Ra` family `Ra` (en's credential), back to suffix `Ra` under strict `ko` + `en`; strict `ja` reads `Gen Hoshino` given `Gen` (default title), strict `zh` `De Xu` given `De`, family `Xu` (default family `De Xu`). Every example the 2026-10-30 entry gives parses on the built code as it says. - The default does not move: tests/v2/test_snapshot.py passes unchanged, a plain choice compiling as before. diff --git a/docs/usage.rst b/docs/usage.rst index cf7df97df..1ab00fa79 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1454,7 +1454,8 @@ Add ``--locale`` to parse with a locale pack (for example ``--locale ru``); see :doc:`locales`. A code no pack has that is a language whose vocabulary file holds readings (``--locale ko``) chooses that language alone; one whose file holds none yet (``vi``) needs ``--strict``, and -a region of a pack's language (``zh-CN``) exits naming the pack to use. +a pack's language with a region or script subtag (``zh-CN``, +``zh-Hant``) exits naming the pack to use. ``--strict`` reads only that language's words (see :ref:`strict-choice`), and needs ``--locale``: diff --git a/nameparser/__main__.py b/nameparser/__main__.py index 598a9d039..4b3bacb7e 100644 --- a/nameparser/__main__.py +++ b/nameparser/__main__.py @@ -81,7 +81,8 @@ def main(argv: list[str] | None = None) -> int: if args.locale is not None: from nameparser import locales # a tag spelled for a pack's code is that pack (pt-BR is pt_br); - # a region of a pack's language (zh-CN, es-MX, ja-JP) is refused + # a pack's language with a region or script subtag (zh-CN, + # zh-Hant, es-MX, ja-JP) is refused # naming the pack, never chosen bare: a bare choice would drop # the pack's list, policy and segmenter in silence code = args.locale.replace("-", "_").casefold() @@ -92,8 +93,8 @@ def main(argv: list[str] | None = None) -> int: except KeyError as exc: unknown = exc.args[0] if language != code and language in locales.available(): - ap.error(f"{unknown}; {args.locale!r} is a region of " - f"the language of the pack {language!r}: use " + ap.error(f"{unknown}; {args.locale!r} names the language " + f"of the pack {language!r} with a subtag: use " f"--locale {language}") # not a pack: a language tag chooses that language alone, # which parser_for checks below; if it is neither, the error diff --git a/tests/v2/test_cli.py b/tests/v2/test_cli.py index 74e39f448..4e850f00a 100644 --- a/tests/v2/test_cli.py +++ b/tests/v2/test_cli.py @@ -321,11 +321,13 @@ def test_cli_strict_with_a_pack_whose_code_is_no_language( # a bare choice would drop the pack's segmentation, two family # names and given-name list, and the segmenter, in silence ("zh-CN", "zh"), ("es-MX", "es"), ("ja-JP", "ja"), ("pt-PT", "pt"), + # a script subtag is not a region, and the message says neither + ("zh-Hant", "zh"), ]) -def test_cli_a_region_of_a_packs_language_names_the_pack( +def test_cli_a_subtag_of_a_packs_language_names_the_pack( capsys: pytest.CaptureFixture[str], tag: str, pack: str) -> None: err = _usage_error(capsys, "--locale", tag) - assert f"use --locale {pack}" in err + assert f"use --locale {pack}" in err and "region" not in err @pytest.mark.parametrize("tag", ["pt-BR", "pt_br", "PT-BR"])