From c874ed6cacb35f4c94786a264678b2020b3fb3d4 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Fri, 31 Jul 2026 15:35:12 +0300 Subject: [PATCH 1/2] Document and enhance the alias system --- babel/localedata.py | 6 ++++++ tests/test_localedata.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/babel/localedata.py b/babel/localedata.py index 297955b97..1967856f5 100644 --- a/babel/localedata.py +++ b/babel/localedata.py @@ -217,6 +217,8 @@ def merge(dict1: MutableMapping[Any, Any], dict2: Mapping[Any, Any]) -> None: if val1 is None: val1 = {} if isinstance(val1, Alias): + # A dict overriding an alias becomes an `(alias, overrides)` tuple + # resolved in `Alias.resolve` or `LocaleDataDict.__getitem__`. val1 = (val1, val2) elif isinstance(val1, tuple): alias, others = val1 @@ -260,6 +262,10 @@ def resolve(self, data: Mapping[str | int | None, Any]) -> Mapping[str | int | N elif isinstance(data, tuple): alias, others = data data = alias.resolve(base) + if others: # Apply overrides on a copy. + data = data.copy() + merge(data, others) + return data diff --git a/tests/test_localedata.py b/tests/test_localedata.py index 20093c7f4..0e500be4a 100644 --- a/tests/test_localedata.py +++ b/tests/test_localedata.py @@ -59,6 +59,24 @@ def test_merge_with_alias_and_resolve(): assert d1['y'] == (alias, {'b': 22, 'e': 25}) +def test_alias_resolving_to_partially_overridden_alias(): + # An alias chain that lands on an `(alias, overrides)` tuple + # must apply the overrides on top of the resolved target. + x_alias = localedata.Alias(('x',)) + data = { + 'x': {'a': 1, 'b': 2}, + 'y': (x_alias, {'b': 22, 'c': 33}), # 'x', partially overridden + 'z': localedata.Alias(('y',)), # an alias landing on the tuple + } + expected = {'a': 1, 'b': 22, 'c': 33} + assert localedata.Alias(('z',)).resolve(data) == expected + d = localedata.LocaleDataDict(data) + assert dict(d['z']) == expected + # Ensure original data is as before: + assert data['x'] == {'a': 1, 'b': 2} + assert data['y'] == (x_alias, {'b': 22, 'c': 33}) + + def test_load(): assert localedata.load('en_US')['languages']['sv'] == 'Swedish' assert localedata.load('en_US') is localedata.load('en_US') From fcb73a0cc11c19f7da700dc69e0a55ed97ef3ec1 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Fri, 31 Jul 2026 12:56:06 +0300 Subject: [PATCH 2/2] Import aliases from CLDR data and use them better We had some manual fallbacks for missing field/unit lengths, and ignored many ``es in CLDR data entirely. This PR imports those aliases and uses them to resolve missing lengths, improving output in corner cases and fixes some rare KeyError crashes too. There's a quirk related to person-variant unit aliases, which are only expressed in CLDR for the short length. If one follows the spec to the letter, a request for a long/narrow person-variant unit will route through the short length and lose the requested length. We use the same "workaround" as ICU to preserve the length, by importing these as whole-unit aliases. Not totally spec- compliant, but the end result is simply better. Fixes #1076 --- babel/dates.py | 11 +---- babel/units.py | 21 +++------ scripts/import_cldr.py | 97 +++++++++++++++++++++++++++++++++++++++++- tests/test_dates.py | 16 +++++++ tests/test_numbers.py | 8 ++++ tests/test_units.py | 39 ++++++++++++++++- 6 files changed, 165 insertions(+), 27 deletions(-) diff --git a/babel/dates.py b/babel/dates.py index 72f2c4439..1f6295783 100644 --- a/babel/dates.py +++ b/babel/dates.py @@ -992,20 +992,13 @@ def _iter_patterns(a_unit): if add_direction: # Try to find the length variant version first ("year-narrow") # before falling back to the default. - unit_rel_patterns = date_fields.get(f"{a_unit}-{format}") or date_fields[a_unit] + unit_rel_patterns = date_fields.get(f"{a_unit}-{format}") or date_fields.get(a_unit) or {} if seconds >= 0: yield unit_rel_patterns['future'] else: yield unit_rel_patterns['past'] a_unit = f"duration-{a_unit}" - unit_pats = unit_patterns.get(a_unit, {}) - yield unit_pats.get(format) - # We do not support `` tags at all while ingesting CLDR data, - # so these aliases specified in `root.xml` are hard-coded here: - # - # - if format in ("long", "narrow"): - yield unit_pats.get("short") + yield unit_patterns.get(a_unit, {}).get(format) # resolves aliases for unit, secs_per_unit in TIMEDELTA_UNITS: value = abs(seconds) / secs_per_unit diff --git a/babel/units.py b/babel/units.py index fa41260af..0f32014aa 100644 --- a/babel/units.py +++ b/babel/units.py @@ -174,21 +174,12 @@ def format_unit( ) plural_form = locale.plural_form(value) - unit_patterns = locale._data["unit_patterns"][q_unit] - - # We do not support `` tags at all while ingesting CLDR data, - # so these aliases specified in `root.xml` are hard-coded here: - # - # - lengths_to_check = [length, "short"] if length in ("long", "narrow") else [length] - - for real_length in lengths_to_check: - length_patterns = unit_patterns.get(real_length, {}) - # Fall back from the correct plural form to "other" - # (this is specified in LDML "Lateral Inheritance") - pat = length_patterns.get(plural_form) or length_patterns.get("other") - if pat: - return pat.format(formatted_value) + length_patterns = locale._data["unit_patterns"][q_unit].get(length, {}) # resolves aliases + # Fall back from the correct plural form to "other" + # (this is specified in LDML "Lateral Inheritance") + pat = length_patterns.get(plural_form) or length_patterns.get("other") + if pat: + return pat.format(formatted_value) # Fall back to a somewhat bad representation. # nb: This is marked as no-cover, as the current CLDR seemingly has no way for this to happen. diff --git a/scripts/import_cldr.py b/scripts/import_cldr.py index 2bbdad173..02e01f199 100755 --- a/scripts/import_cldr.py +++ b/scripts/import_cldr.py @@ -78,6 +78,19 @@ def _translate_alias(ctxt, path): return keys +def _sibling_alias_type(alias_elem) -> str: + """ + Extract target type from an alias path pointing at a sibling. + + E.g. ``../unit[@type='duration-year']`` -> ``duration-year``. + """ + path_attr = alias_elem.attrib['path'] + parent, _, leaf = path_attr.rpartition('/') + if parent != "..": + raise ValueError(f"not a sibling alias: {path_attr}") + return TYPE_ATTR_RE.match(leaf).group(1) + + def _parse_currency_date(s): if not s: return None @@ -805,8 +818,20 @@ def parse_decimal_formats(data, tree): length_type = elem.attrib.get('type') if _should_skip_elem(elem, length_type, decimal_formats): continue - if elem.findall('./alias'): - # TODO map the alias to its target + alias_elem = elem.find('./alias') + if alias_elem is not None: + if alias_elem.attrib['path'] != "../decimalFormatLength[@type='short']": + raise ValueError("Unexpected decimal format alias in new CLDR data; this may require hand-holding.") + # In particular: if you're reading this, the tribal knowledge below + # (that this is compact_decimal_format) needs to be double-checked. + + # `root.xml` aliases `long` to `short`. + # The typed lengths only carry compact patterns, so the alias lands in `compact_decimal_formats`. + # Only root has this. + compact_decimal_formats = data.setdefault('compact_decimal_formats', {}) + compact_decimal_formats[length_type] = Alias( + ('compact_decimal_formats', _sibling_alias_type(alias_elem)), + ) continue for pattern_el in elem.findall('./decimalFormat/pattern'): pattern_type = pattern_el.attrib.get('type') @@ -879,11 +904,58 @@ def parse_unit_patterns(data, tree): unit_patterns = data.setdefault('unit_patterns', {}) compound_patterns = data.setdefault('compound_unit_patterns', {}) unit_display_names = data.setdefault('unit_display_names', {}) + # `root.xml` aliases the `long` and `narrow` unit lengths to `short`, + # and some units to others (e.g. `duration-day-person` to `duration-day`). + # This is keyed unit-first instead of length-first (XML), + # so length aliases are expanded into per-unit aliases below. + # Only root has aliases. + length_aliases = {} for elem in tree.findall('.//units/unitLength'): unit_length_type = elem.attrib['type'] + alias_elem = elem.find('./alias') + if alias_elem is not None: + length_aliases[unit_length_type] = _sibling_alias_type(alias_elem) + continue for unit in elem.findall('unit'): unit_type = unit.attrib['type'] + alias_elem = unit.find('./alias') + if alias_elem is not None: + target_unit = _sibling_alias_type(alias_elem) + if unit_type.endswith('-person'): + # HACK (and this is a mouthful): + # The `duration-*-person` units (used for formatting ages) are + # aliased to their base units. The alias element only exists + # inside `unitLength[short]`, so literal resolution would route + # a long/narrow request through the short chain and lose the + # requested length ("3 y" where "3 years" was wanted). + # ICU considers this a deficiency of the alias data structure + # (https://unicode-org.atlassian.net/browse/ICU-20400) and + # compensates by stripping the `-person` suffix before lookup + # This seems to make sense, so we do that too: + # alias the whole unit, preserving the requested length. + # + # NB: this deliberately bends literal TR35 alias resolution to + # match ICU's output. Should CLDR ever gain a length-preserving + # alias representation, spec compliance is restored by deleting + # this special-case `-person` branch and translating those + # aliases as written, as the branch below does.. + unit_patterns[unit_type] = Alias(('unit_patterns', target_unit)) + unit_display_names[unit_type] = Alias(('unit_display_names', target_unit)) + else: + # The other aliased units (`graphics-dot*` -> `graphics-pixel*`, + # `energy-foodcalorie` -> `energy-kilocalorie`) get no such + # compensation in ICU and follow the literal chain: the alias + # applies within this length only, and a locale's own data for + # the unit at other lengths (plus the length aliases below) + # takes precedence over the target unit's. + unit_patterns.setdefault(unit_type, {})[unit_length_type] = Alias( + ('unit_patterns', target_unit, unit_length_type), + ) + unit_display_names.setdefault(unit_type, {})[unit_length_type] = Alias( + ('unit_display_names', target_unit, unit_length_type), + ) + continue unit_and_length_patterns = unit_patterns.setdefault(unit_type, {}).setdefault(unit_length_type, {}) for pattern in unit.findall('unitPattern'): if pattern.attrib.get('case', 'nominative') != 'nominative': @@ -922,11 +994,32 @@ def parse_unit_patterns(data, tree): compound_unit_info['compound_variations'] = compound_variations compound_patterns.setdefault(unit_type, {})[unit_length_type] = compound_unit_info + for aliased_length, target_length in length_aliases.items(): + for dict_name, dst in ( + ('unit_patterns', unit_patterns), + ('compound_unit_patterns', compound_patterns), + ('unit_display_names', unit_display_names), + ): + for unit_type, lengths in dst.items(): + if isinstance(lengths, Alias): + # A length-preserving whole-unit alias installed above. + continue + lengths.setdefault(aliased_length, Alias((dict_name, unit_type, target_length))) + def parse_date_fields(data, tree): date_fields = data.setdefault('date_fields', {}) for elem in tree.findall('.//dates/fields/field'): field_type = elem.attrib['type'] + # `root` aliases `x-narrow` to `x-short` and `x-short` to `x`. + # Locales defining only some length variants inherit these. + # Only root has these. + alias_elem = elem.find('alias') + if alias_elem is not None: + date_fields[field_type] = Alias( + _translate_alias(['date_fields', field_type], alias_elem.attrib['path']), + ) + continue date_fields.setdefault(field_type, {}) for rel_time in elem.findall('relativeTime'): rel_time_type = rel_time.attrib['type'] diff --git a/tests/test_dates.py b/tests/test_dates.py index f80bb3288..0cc072990 100644 --- a/tests/test_dates.py +++ b/tests/test_dates.py @@ -1190,6 +1190,22 @@ def test_issue_1162(locale, format, negative, expected): assert dates.format_timedelta(delta, add_direction=True, format=format, locale=locale) == expected +@pytest.mark.parametrize(('locale', 'format', 'expected'), [ + # `am` defines `year-short` with names but no patterns; this used to raise KeyError. + ('am', 'short', 'በ2 ዓመታት ውስጥ'), + ('am', 'narrow', 'በ2 ዓመታት ውስጥ'), + # `af` has no `year-narrow`; root aliases narrow to short, + # so the narrow form must use `year-short` ('oor 2 j.') + # rather than skipping straight to `year`. + ('af', 'narrow', 'oor 2 j.'), + ('af', 'short', 'oor 2 j.'), + ('af', 'long', 'oor 2 jaar'), +]) +def test_issue_1076_date_field_length_aliases(locale, format, expected): + delta = timedelta(days=800) + assert dates.format_timedelta(delta, add_direction=True, format=format, locale=locale) == expected + + def test_issue_1192(): # The actual returned value here is not actually strictly specified ("get_timezone_name" # is not an operation specified as such). Issue #1192 concerned this invocation returning diff --git a/tests/test_numbers.py b/tests/test_numbers.py index cf888b7e6..4060da2ae 100644 --- a/tests/test_numbers.py +++ b/tests/test_numbers.py @@ -673,3 +673,11 @@ def test_format_decimal_with_none_locale(monkeypatch): monkeypatch.setattr(numbers, "LC_NUMERIC", None) # Pretend we couldn't find any locale when importing the module with pytest.raises(TypeError, match="Empty"): numbers.format_decimal(0, locale=None) + + +def test_issue_1076_fixes(): + # Japanese only defines short compact decimal formats; root aliases long to short. This used to raise a KeyError. + assert numbers.format_compact_decimal(12345, format_type="long", locale="ja") == "1万" + assert numbers.format_compact_decimal(2345678, format_type="long", locale="ja") == "235万" + # Swahili does not define an accounting currency format; root aliases it to the standard format. + assert numbers.format_currency(-2, "USD", format_type="accounting", locale="sw") == "-US$\xa02.00" diff --git a/tests/test_units.py b/tests/test_units.py index cab9c0a1b..b701beffb 100644 --- a/tests/test_units.py +++ b/tests/test_units.py @@ -1,6 +1,6 @@ import pytest -from babel.units import format_unit +from babel.units import format_unit, get_unit_name # New units in CLDR 46 @@ -35,3 +35,40 @@ def test_deprecated_unit_ids(): for id in ("concentr-permillion", "concentr-portion", "concentr-portion-per-1e9"): with pytest.warns(DeprecationWarning, match=id): format_unit(1, id, locale='en') + + +@pytest.mark.parametrize('count, unit, locale, length, expected', [ + # Root aliases `duration-*-person` to `duration-*`; + # no locale defines the person variants at all. + # These resolve length-preservingly (person long -> base long), + # matching ICU's `-person`-stripping behavior + # (see `getMeasureData` in https://github.com/unicode-org/icu/blob/main/icu4c/source/i18n/number_longnames.cpp + # and https://unicode-org.atlassian.net/browse/ICU-20400). + # This deliberately bends literal TR35 alias resolution. + # See `parse_unit_patterns` in `scripts/import_cldr.py` for the full story. + (2, 'duration-day-person', 'af', 'long', '2 dae'), + (3, 'duration-day-person', 'fi', 'long', '3 päivää'), + (3, 'duration-day-person', 'fi', 'short', '3 pv'), + (3, 'duration-day-person', 'fi', 'narrow', '3pv'), + (3, 'duration-year-person', 'fi', 'long', '3 vuotta'), + # `fi` defines `energy-foodcalorie` at long and narrow but not short; + # the root alias fills short in from `energy-kilocalorie`. + (3, 'energy-foodcalorie', 'fi', 'short', '3 kcal'), + # `fi` defines `graphics-dot` at short and narrow but not long; + # the root alias fills long in from short. + (3, 'graphics-dot', 'fi', 'long', '3 pistettä'), + # `cs` defines `graphics-dot` itself but not its short forms; + # the root aliases short to `graphics-pixel` short rather than the display name. + (3, 'graphics-dot', 'cs', 'short', '3 px'), +]) +def test_issue_1076_unit_aliases(count, unit, locale, length, expected): + assert format_unit(count, unit, length, locale=locale) == expected + + +def test_issue_1076_unit_name_length_aliases(): + # Finnish defines no narrow display name for days; root aliases narrow to + # short. This used to return None. + assert get_unit_name('duration-day', length='narrow', locale='fi') == 'pv' + # The person variant resolves length-preservingly to `duration-day`. + assert get_unit_name('duration-day-person', length='long', locale='fi') == 'päivät' + assert get_unit_name('duration-day-person', length='short', locale='fi') == 'pv'