icu_normalizer/lib.rs
1// This file is part of ICU4X. For terms of use, please see the file
2// called LICENSE at the top level of the ICU4X source tree
3// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5// https://github.com/unicode-org/icu4x/blob/main/documents/process/boilerplate.md#library-annotations
6#![cfg_attr(not(any(test, doc)), no_std)]
7#![cfg_attr(
8 not(test),
9 deny(
10 clippy::indexing_slicing,
11 clippy::unwrap_used,
12 clippy::expect_used,
13 clippy::panic,
14 )
15)]
16#![warn(missing_docs)]
17
18//! Normalizing text into Unicode Normalization Forms.
19//!
20//! This module is published as its own crate ([`icu_normalizer`](https://docs.rs/icu_normalizer/latest/icu_normalizer/))
21//! and as part of the [`icu`](https://docs.rs/icu/latest/icu/) crate. See the latter for more details on the ICU4X project.
22//!
23//! # Functionality
24//!
25//! The top level of the crate provides normalization of input into the four normalization forms defined in [UAX #15: Unicode
26//! Normalization Forms](https://www.unicode.org/reports/tr15/): NFC, NFD, NFKC, and NFKD.
27//!
28//! Three kinds of contiguous inputs are supported: known-well-formed UTF-8 (`&str`), potentially-not-well-formed UTF-8,
29//! and potentially-not-well-formed UTF-16. Additionally, an iterator over `char` can be wrapped in a normalizing iterator.
30//!
31//! The `uts46` module provides the combination of mapping and normalization operations for [UTS #46: Unicode IDNA
32//! Compatibility Processing](https://www.unicode.org/reports/tr46/). This functionality is not meant to be used by
33//! applications directly. Instead, it is meant as a building block for a full implementation of UTS #46, such as the
34//! [`idna`](https://docs.rs/idna/latest/idna/) crate.
35//!
36//! The `properties` module provides the non-recursive canonical decomposition operation on a per `char` basis and
37//! the canonical compositon operation given two `char`s. It also provides access to the Canonical Combining Class
38//! property. These operations are primarily meant for [HarfBuzz](https://harfbuzz.github.io/), the types
39//! [`CanonicalComposition`](properties::CanonicalComposition), [`CanonicalDecomposition`](properties::CanonicalDecomposition),
40//! and [`CanonicalCombiningClassMap`](properties::CanonicalCombiningClassMap) implement the [`harfbuzz_traits`] if
41//! the `harfbuzz_traits` Cargo feature is enabled.
42//!
43//! Notably, this normalizer does _not_ provide the normalization “quick check” that can result in “maybe” in
44//! addition to “yes” and “no”. The normalization checks provided by this crate always give a definitive
45//! non-“maybe” answer.
46//!
47//! # Examples
48//!
49//! ```
50//! let nfc = icu_normalizer::ComposingNormalizerBorrowed::new_nfc();
51//! assert_eq!(nfc.normalize("a\u{0308}"), "ä");
52//! assert!(nfc.is_normalized("ä"));
53//!
54//! let nfd = icu_normalizer::DecomposingNormalizerBorrowed::new_nfd();
55//! assert_eq!(nfd.normalize("ä"), "a\u{0308}");
56//! assert!(!nfd.is_normalized("ä"));
57//! ```
58
59extern crate alloc;
60
61// TODO: The plan is to replace
62// `#[cfg(not(icu4x_unstable_fast_trie_only))]`
63// with
64// `#[cfg(feature = "serde")]`
65// and
66// `#[cfg(icu4x_unstable_fast_trie_only)]`
67// with
68// `#[cfg(not(feature = "serde"))]`
69//
70// Before doing so:
71// * The type of the UTS 46 trie needs to be
72// disentangled from the type of the NFD/NFKD tries.
73// This will involve a more generic iterator hidden
74// inside the public iterator types.
75// * datagen needs to emit fast-mode tries for the
76// NFD and NFKD tries.
77// * The markers and possibly the data struct type
78// for NFD and NFKD need to be revised per policy.
79
80#[cfg(not(icu4x_unstable_fast_trie_only))]
81type Trie<'trie> = CodePointTrie<'trie, u32>;
82
83#[cfg(icu4x_unstable_fast_trie_only)]
84type Trie<'trie> = FastCodePointTrie<'trie, u32>;
85
86#[cfg(feature = "harfbuzz_traits")]
87mod harfbuzz;
88pub mod properties;
89pub mod provider;
90pub mod uts46;
91
92use crate::provider::CanonicalCompositions;
93use crate::provider::DecompositionData;
94use crate::provider::NormalizerNfdDataV1;
95use crate::provider::NormalizerNfkdDataV1;
96use crate::provider::NormalizerUts46DataV1;
97use alloc::borrow::Cow;
98use alloc::string::String;
99use icu_collections::char16trie::Char16Trie;
100#[cfg(not(icu4x_unstable_fast_trie_only))]
101use icu_collections::codepointtrie::CodePointTrie;
102#[cfg(icu4x_unstable_fast_trie_only)]
103use icu_collections::codepointtrie::FastCodePointTrie;
104#[cfg(icu4x_unstable_fast_trie_only)]
105use icu_collections::codepointtrie::TypedCodePointTrie;
106#[cfg(feature = "icu_properties")]
107use icu_properties::props::CanonicalCombiningClass;
108use icu_provider::prelude::*;
109use provider::DecompositionTables;
110use provider::NormalizerNfcV1;
111use provider::NormalizerNfdTablesV1;
112use provider::NormalizerNfkdTablesV1;
113use smallvec::SmallVec;
114#[cfg(feature = "utf8_iter")]
115use utf8_iter::Utf8CharsEx;
116#[cfg(feature = "utf16_iter")]
117use utf16_iter::Utf16CharsEx;
118use zerovec::{ZeroSlice, zeroslice};
119
120// The optimizations in the area where `likely` is used
121// are extremely brittle. `likely` is useful in the typed-trie
122// case on the UTF-16 fast path, but in order not to disturb
123// the untyped-trie case on the UTF-16 fast path, make the
124// annotations no-ops in the untyped-trie case.
125
126// `cold_path` and `likely` come from
127// https://github.com/rust-lang/hashbrown/commit/64bd7db1d1b148594edfde112cdb6d6260e2cfc3 .
128// See https://github.com/rust-lang/hashbrown/commit/64bd7db1d1b148594edfde112cdb6d6260e2cfc3#commitcomment-164768806
129// for permission to relicense under Unicode-3.0.
130
131#[cfg(all(icu4x_unstable_fast_trie_only, feature = "utf16_iter"))]
132#[inline(always)]
133#[cold]
134fn cold_path() {}
135
136#[cfg(all(icu4x_unstable_fast_trie_only, feature = "utf16_iter"))]
137#[inline(always)]
138pub(crate) fn likely(b: bool) -> bool {
139 if b {
140 true
141 } else {
142 cold_path();
143 false
144 }
145}
146
147// End import from https://github.com/rust-lang/hashbrown/commit/64bd7db1d1b148594edfde112cdb6d6260e2cfc3 .
148
149/// No-op for typed trie case.
150#[cfg(all(not(icu4x_unstable_fast_trie_only), feature = "utf16_iter"))]
151#[inline(always)]
152fn likely(b: bool) -> bool {
153 b
154}
155
156// This type exists as a shim for `icu_properties` `CanonicalCombiningClass` when the crate is disabled
157// It should not be exposed to users.
158#[cfg(not(feature = "icu_properties"))]
159#[derive(Copy, Clone, Eq, PartialEq, PartialOrd, Ord)]
160struct CanonicalCombiningClass(u8);
161
162#[cfg(not(feature = "icu_properties"))]
163#[allow(non_upper_case_globals)]
164impl CanonicalCombiningClass {
165 // See https://www.unicode.org/reports/tr44/#Canonical_Combining_Class_Values
166 const NotReordered: Self = Self(0);
167 const Above: Self = Self(230);
168 const KanaVoicing: Self = Self(8);
169}
170
171/// Treatment of the ignorable marker (0xFFFFFFFF) in data.
172#[derive(Debug, PartialEq, Eq)]
173enum IgnorableBehavior {
174 /// 0xFFFFFFFF in data is not supported.
175 Unsupported,
176 /// Ignorables are ignored.
177 Ignored,
178 /// Ignorables are treated as singleton decompositions
179 /// to the REPLACEMENT CHARACTER.
180 ReplacementCharacter,
181}
182
183/// Marker for UTS 46 ignorables.
184///
185/// See trie-value-format.md
186const IGNORABLE_MARKER: u32 = 0xFFFFFFFF;
187
188/// Marker that the decomposition does not round trip via NFC.
189///
190/// See trie-value-format.md
191const NON_ROUND_TRIP_MARKER: u32 = 1 << 30;
192
193/// Marker that the first character of the decomposition
194/// can combine backwards.
195///
196/// See trie-value-format.md
197const BACKWARD_COMBINING_MARKER: u32 = 1 << 31;
198
199/// Mask for the bits have to be zero for this to be a BMP
200/// singleton decomposition, or value baked into the surrogate
201/// range.
202///
203/// See trie-value-format.md
204const HIGH_ZEROS_MASK: u32 = 0x3FFF0000;
205
206/// Mask for the bits have to be zero for this to be a complex
207/// decomposition.
208///
209/// See trie-value-format.md
210const LOW_ZEROS_MASK: u32 = 0xFFE0;
211
212/// Checks if a trie value carries a (non-zero) canonical
213/// combining class.
214///
215/// See trie-value-format.md
216#[inline]
217fn trie_value_has_ccc(trie_value: u32) -> bool {
218 (trie_value & 0x3FFFFE00) == 0xD800
219}
220
221/// Checks if the trie signifies a special non-starter decomposition.
222///
223/// See trie-value-format.md
224fn trie_value_indicates_special_non_starter_decomposition(trie_value: u32) -> bool {
225 (trie_value & 0x3FFFFF00) == 0xD900
226}
227
228/// Checks if a trie value signifies a character whose decomposition
229/// starts with a non-starter.
230///
231/// See trie-value-format.md
232fn decomposition_starts_with_non_starter(trie_value: u32) -> bool {
233 trie_value_has_ccc(trie_value)
234}
235
236/// Extracts a canonical combining class (possibly zero) from a trie value.
237///
238/// See trie-value-format.md
239fn ccc_from_trie_value(trie_value: u32) -> CanonicalCombiningClass {
240 if trie_value_has_ccc(trie_value) {
241 CanonicalCombiningClass(trie_value as u8)
242 } else {
243 CanonicalCombiningClass::NotReordered
244 }
245}
246
247/// The tail (everything after the first character) of the NFKD form U+FDFA
248/// as 16-bit units.
249static FDFA_NFKD: [u16; 17] = [
250 0x644, 0x649, 0x20, 0x627, 0x644, 0x644, 0x647, 0x20, 0x639, 0x644, 0x64A, 0x647, 0x20, 0x648,
251 0x633, 0x644, 0x645,
252];
253
254/// Marker value for U+FDFA in NFKD. (Unified with Hangul syllable marker,
255/// but they differ by `NON_ROUND_TRIP_MARKER`.)
256///
257/// See trie-value-format.md
258const FDFA_MARKER: u16 = 1;
259
260// These constants originate from page 143 of Unicode 14.0
261/// Syllable base
262const HANGUL_S_BASE: u32 = 0xAC00;
263/// Lead jamo base
264const HANGUL_L_BASE: u32 = 0x1100;
265/// Vowel jamo base
266const HANGUL_V_BASE: u32 = 0x1161;
267/// Trail jamo base (deliberately off by one to account for the absence of a trail)
268const HANGUL_T_BASE: u32 = 0x11A7;
269/// Lead jamo count
270const HANGUL_L_COUNT: u32 = 19;
271/// Vowel jamo count
272const HANGUL_V_COUNT: u32 = 21;
273/// Trail jamo count (deliberately off by one to account for the absence of a trail)
274const HANGUL_T_COUNT: u32 = 28;
275/// Vowel jamo count times trail jamo count
276const HANGUL_N_COUNT: u32 = 588;
277/// Syllable count
278const HANGUL_S_COUNT: u32 = 11172;
279
280/// One past the conjoining jamo block
281const HANGUL_JAMO_LIMIT: u32 = 0x1200;
282
283/// If `opt` is `Some`, unwrap it. If `None`, panic if debug assertions
284/// are enabled and return `default` if debug assertions are not enabled.
285///
286/// Use this only if the only reason why `opt` could be `None` is bogus
287/// data from the provider.
288#[inline(always)]
289fn unwrap_or_gigo<T>(opt: Option<T>, default: T) -> T {
290 if let Some(val) = opt {
291 val
292 } else {
293 // GIGO case
294 debug_assert!(false);
295 default
296 }
297}
298
299/// Convert a `u32` _obtained from data provider data_ to `char`.
300#[inline(always)]
301fn char_from_u32(u: u32) -> char {
302 unwrap_or_gigo(char::from_u32(u), char::REPLACEMENT_CHARACTER)
303}
304
305/// Convert a `u16` _obtained from data provider data_ to `char`.
306#[inline(always)]
307fn char_from_u16(u: u16) -> char {
308 char_from_u32(u32::from(u))
309}
310
311const EMPTY_U16: &ZeroSlice<u16> = zeroslice![];
312
313const EMPTY_CHAR: &ZeroSlice<char> = zeroslice![];
314
315#[inline(always)]
316fn in_inclusive_range(c: char, start: char, end: char) -> bool {
317 u32::from(c).wrapping_sub(u32::from(start)) <= (u32::from(end) - u32::from(start))
318}
319
320#[inline(always)]
321#[cfg(feature = "utf16_iter")]
322fn in_inclusive_range16(u: u16, start: u16, end: u16) -> bool {
323 u.wrapping_sub(start) <= (end - start)
324}
325
326/// Performs canonical composition (including Hangul) on a pair of
327/// characters or returns `None` if these characters don't compose.
328/// Composition exclusions are taken into account.
329#[inline]
330fn compose(canonical_compositions: &Char16Trie, starter: char, second: char) -> Option<char> {
331 let v = u32::from(second).wrapping_sub(HANGUL_V_BASE);
332 if v >= HANGUL_JAMO_LIMIT - HANGUL_V_BASE {
333 return compose_non_hangul(canonical_compositions, starter, second);
334 }
335 if v < HANGUL_V_COUNT {
336 let l = u32::from(starter).wrapping_sub(HANGUL_L_BASE);
337 if l < HANGUL_L_COUNT {
338 let lv = l * HANGUL_N_COUNT + v * HANGUL_T_COUNT;
339 // Safe, because the inputs are known to be in range.
340 return Some(unsafe { char::from_u32_unchecked(HANGUL_S_BASE + lv) });
341 }
342 return None;
343 }
344 if in_inclusive_range(second, '\u{11A8}', '\u{11C2}') {
345 let lv = u32::from(starter).wrapping_sub(HANGUL_S_BASE);
346 if lv < HANGUL_S_COUNT && lv % HANGUL_T_COUNT == 0 {
347 let lvt = lv + (u32::from(second) - HANGUL_T_BASE);
348 // Safe, because the inputs are known to be in range.
349 return Some(unsafe { char::from_u32_unchecked(HANGUL_S_BASE + lvt) });
350 }
351 }
352 None
353}
354
355/// Performs (non-Hangul) canonical composition on a pair of characters
356/// or returns `None` if these characters don't compose. Composition
357/// exclusions are taken into account.
358fn compose_non_hangul(
359 canonical_compositions: &Char16Trie,
360 starter: char,
361 second: char,
362) -> Option<char> {
363 // To make the trie smaller, the pairs are stored second character first.
364 // Given how this method is used in ways where it's known that `second`
365 // is or isn't a starter. We could potentially split the trie into two
366 // tries depending on whether `second` is a starter.
367 let mut cursor = canonical_compositions.cursor();
368 cursor.step(second);
369 if cursor.is_empty() {
370 return None;
371 }
372 if cursor.value().is_some() {
373 // GIGO case
374 debug_assert!(false);
375 return None;
376 }
377 cursor.step(starter);
378 let i = cursor.value()?;
379 if !cursor.is_empty() {
380 // GIGO case
381 debug_assert!(false);
382 return None;
383 }
384 if let Some(c) = char::from_u32(i as u32) {
385 Some(c)
386 } else {
387 // GIGO case
388 debug_assert!(false);
389 None
390 }
391}
392
393/// See trie-value-format.md
394#[inline(always)]
395fn starter_and_decomposes_to_self_impl(trie_val: u32) -> bool {
396 // The REPLACEMENT CHARACTER has `NON_ROUND_TRIP_MARKER` set,
397 // and this function needs to ignore that.
398 (trie_val & !(BACKWARD_COMBINING_MARKER | NON_ROUND_TRIP_MARKER)) == 0
399}
400
401/// See trie-value-format.md
402#[inline(always)]
403fn potential_passthrough_and_cannot_combine_backwards_impl(trie_val: u32) -> bool {
404 (trie_val & (NON_ROUND_TRIP_MARKER | BACKWARD_COMBINING_MARKER)) == 0
405}
406
407/// Struct for holding together a character and the value
408/// looked up for it from the NFD trie in a more explicit
409/// way than an anonymous pair.
410/// Also holds a flag about the supplementary-trie provenance.
411#[derive(Debug, PartialEq, Eq)]
412struct CharacterAndTrieValue {
413 character: char,
414 /// See trie-value-format.md
415 trie_val: u32,
416}
417
418impl CharacterAndTrieValue {
419 #[inline(always)]
420 pub fn new(c: char, trie_value: u32) -> Self {
421 CharacterAndTrieValue {
422 character: c,
423 trie_val: trie_value,
424 }
425 }
426
427 #[inline(always)]
428 pub fn starter_and_decomposes_to_self(&self) -> bool {
429 starter_and_decomposes_to_self_impl(self.trie_val)
430 }
431
432 /// See trie-value-format.md
433 #[inline(always)]
434 #[cfg(feature = "utf8_iter")]
435 pub fn starter_and_decomposes_to_self_except_replacement(&self) -> bool {
436 // This intentionally leaves `NON_ROUND_TRIP_MARKER` in the value
437 // to be compared with zero. U+FFFD has that flag set despite really
438 // being being round-tripping in order to make UTF-8 errors
439 // ineligible for passthrough.
440 (self.trie_val & !BACKWARD_COMBINING_MARKER) == 0
441 }
442
443 /// See trie-value-format.md
444 #[inline(always)]
445 pub fn can_combine_backwards(&self) -> bool {
446 (self.trie_val & BACKWARD_COMBINING_MARKER) != 0
447 }
448 /// See trie-value-format.md
449 #[inline(always)]
450 pub fn potential_passthrough(&self) -> bool {
451 (self.trie_val & NON_ROUND_TRIP_MARKER) == 0
452 }
453 /// See trie-value-format.md
454 #[inline(always)]
455 pub fn potential_passthrough_and_cannot_combine_backwards(&self) -> bool {
456 potential_passthrough_and_cannot_combine_backwards_impl(self.trie_val)
457 }
458}
459
460/// Pack a `char` and a `CanonicalCombiningClass` in
461/// 32 bits (the former in the lower 24 bits and the
462/// latter in the high 8 bits). The latter can be
463/// initialized to 0xFF upon creation, in which case
464/// it can be actually set later by calling
465/// `set_ccc_from_trie_if_not_already_set`. This is
466/// a micro optimization to avoid the Canonical
467/// Combining Class trie lookup when there is only
468/// one combining character in a sequence. This type
469/// is intentionally non-`Copy` to get compiler help
470/// in making sure that the class is set on the
471/// instance on which it is intended to be set
472/// and not on a temporary copy.
473///
474/// Note that 0xFF is won't be assigned to an actual
475/// canonical combining class per definition D104
476/// in The Unicode Standard.
477//
478// NOTE: The Pernosco debugger has special knowledge
479// of this struct. Please do not change the bit layout
480// or the crate-module-qualified name of this struct
481// without coordination.
482#[derive(Debug)]
483struct CharacterAndClass(u32);
484
485impl CharacterAndClass {
486 pub fn new(c: char, ccc: CanonicalCombiningClass) -> Self {
487 CharacterAndClass(u32::from(c) | (u32::from(ccc.0) << 24))
488 }
489 pub fn new_with_placeholder(c: char) -> Self {
490 CharacterAndClass(u32::from(c) | ((0xFF) << 24))
491 }
492 pub fn new_with_trie_value(c_tv: CharacterAndTrieValue) -> Self {
493 Self::new(c_tv.character, ccc_from_trie_value(c_tv.trie_val))
494 }
495 pub fn new_starter(c: char) -> Self {
496 CharacterAndClass(u32::from(c))
497 }
498 /// This method must exist for Pernosco to apply its special rendering.
499 /// Also, this must not be dead code!
500 pub fn character(&self) -> char {
501 // Safe, because the low 24 bits came from a `char`
502 // originally.
503 unsafe { char::from_u32_unchecked(self.0 & 0xFFFFFF) }
504 }
505 /// This method must exist for Pernosco to apply its special rendering.
506 pub fn ccc(&self) -> CanonicalCombiningClass {
507 CanonicalCombiningClass((self.0 >> 24) as u8)
508 }
509
510 pub fn character_and_ccc(&self) -> (char, CanonicalCombiningClass) {
511 (self.character(), self.ccc())
512 }
513 pub fn set_ccc_from_trie_if_not_already_set(&mut self, trie: &Trie) {
514 if self.0 >> 24 != 0xFF {
515 return;
516 }
517 let scalar = self.0 & 0xFFFFFF;
518 self.0 = ((ccc_from_trie_value(trie.get32_u32(scalar)).0 as u32) << 24) | scalar;
519 }
520}
521
522// This function exists as a borrow check helper.
523#[inline(always)]
524fn sort_slice_by_ccc(slice: &mut [CharacterAndClass], trie: &Trie) {
525 // We don't look up the canonical combining class for starters
526 // of for single combining characters between starters. When
527 // there's more than one combining character between starters,
528 // we look up the canonical combining class for each character
529 // exactly once.
530 if slice.len() < 2 {
531 return;
532 }
533 slice
534 .iter_mut()
535 .for_each(|cc| cc.set_ccc_from_trie_if_not_already_set(trie));
536 slice.sort_by_key(|cc| cc.ccc());
537}
538
539/// An iterator adaptor that turns an `Iterator` over `char` into
540/// a lazily-decomposed `char` sequence.
541#[derive(Debug)]
542pub struct Decomposition<'data, I>
543where
544 I: Iterator<Item = char>,
545{
546 delegate: I,
547 buffer: SmallVec<[CharacterAndClass; 17]>, // Enough to hold NFKD for U+FDFA
548 /// The index of the next item to be read from `buffer`.
549 /// The purpose if this index is to avoid having to move
550 /// the rest upon every read.
551 buffer_pos: usize,
552 // At the start of `next()` if not `None`, this is a pending unnormalized
553 // starter. When `Decomposition` appears alone, this is never a non-starter.
554 // However, when `Decomposition` appears inside a `Composition`, this
555 // may become a non-starter before `decomposing_next()` is called.
556 pending: Option<CharacterAndTrieValue>, // None at end of stream
557 // See trie-value-format.md
558 trie: &'data Trie<'data>,
559 scalars16: &'data ZeroSlice<u16>,
560 scalars24: &'data ZeroSlice<char>,
561 supplementary_scalars16: &'data ZeroSlice<u16>,
562 supplementary_scalars24: &'data ZeroSlice<char>,
563 /// The lowest character for which either of the following does
564 /// not hold:
565 /// 1. Decomposes to self.
566 /// 2. Decomposition starts with a non-starter
567 decomposition_passthrough_bound: u32, // never above 0xC0
568 ignorable_behavior: IgnorableBehavior, // Arguably should be a type parameter
569}
570
571impl<'data, I> Decomposition<'data, I>
572where
573 I: Iterator<Item = char>,
574{
575 /// Constructs a decomposing iterator adapter from a delegate
576 /// iterator and references to the necessary data, without
577 /// supplementary data.
578 ///
579 /// Use `DecomposingNormalizer::normalize_iter()` instead unless
580 /// there's a good reason to use this constructor directly.
581 ///
582 /// Public but hidden in order to be able to use this from the
583 /// collator.
584 #[doc(hidden)] // used in collator
585 pub fn new(
586 delegate: I,
587 decompositions: &'data DecompositionData,
588 tables: &'data DecompositionTables,
589 ) -> Self {
590 Self::new_with_supplements(
591 delegate,
592 decompositions,
593 tables,
594 None,
595 0xC0,
596 IgnorableBehavior::Unsupported,
597 )
598 }
599
600 /// Constructs a decomposing iterator adapter from a delegate
601 /// iterator and references to the necessary data, including
602 /// supplementary data.
603 ///
604 /// Use `DecomposingNormalizer::normalize_iter()` instead unless
605 /// there's a good reason to use this constructor directly.
606 fn new_with_supplements(
607 delegate: I,
608 decompositions: &'data DecompositionData,
609 tables: &'data DecompositionTables,
610 supplementary_tables: Option<&'data DecompositionTables>,
611 decomposition_passthrough_bound: u8,
612 ignorable_behavior: IgnorableBehavior,
613 ) -> Self {
614 let mut ret = Decomposition::<I> {
615 delegate,
616 buffer: SmallVec::new(), // Normalized
617 buffer_pos: 0,
618 // Initialize with a placeholder starter in case
619 // the real stream starts with a non-starter.
620 pending: Some(CharacterAndTrieValue::new('\u{FFFF}', 0)),
621 #[allow(clippy::useless_conversion, clippy::expect_used)] // Expectation always succeeds when untyped tries are in use
622 trie: <&Trie>::try_from(&decompositions.trie).expect("Unexpected trie type in data"),
623 scalars16: &tables.scalars16,
624 scalars24: &tables.scalars24,
625 supplementary_scalars16: if let Some(supplementary) = supplementary_tables {
626 &supplementary.scalars16
627 } else {
628 EMPTY_U16
629 },
630 supplementary_scalars24: if let Some(supplementary) = supplementary_tables {
631 &supplementary.scalars24
632 } else {
633 EMPTY_CHAR
634 },
635 decomposition_passthrough_bound: u32::from(decomposition_passthrough_bound),
636 ignorable_behavior,
637 };
638 let _ = ret.next(); // Remove the U+FFFF placeholder
639 ret
640 }
641
642 fn push_decomposition16(
643 &mut self,
644 offset: usize,
645 len: usize,
646 only_non_starters_in_trail: bool,
647 slice16: &ZeroSlice<u16>,
648 ) -> (char, usize) {
649 let (starter, tail) = slice16
650 .get_subslice(offset..offset + len)
651 .and_then(|slice| slice.split_first())
652 .map_or_else(
653 || {
654 // GIGO case
655 debug_assert!(false);
656 (char::REPLACEMENT_CHARACTER, EMPTY_U16)
657 },
658 |(first, trail)| (char_from_u16(first), trail),
659 );
660 if only_non_starters_in_trail {
661 // All the rest are combining
662 self.buffer.extend(
663 tail.iter()
664 .map(|u| CharacterAndClass::new_with_placeholder(char_from_u16(u))),
665 );
666 (starter, 0)
667 } else {
668 let mut i = 0;
669 let mut combining_start = 0;
670 for u in tail.iter() {
671 let ch = char_from_u16(u);
672 let trie_value = self.trie.get(ch);
673 self.buffer.push(CharacterAndClass::new_with_trie_value(
674 CharacterAndTrieValue::new(ch, trie_value),
675 ));
676 i += 1;
677 // Half-width kana and iota subscript don't occur in the tails
678 // of these multicharacter decompositions.
679 if !decomposition_starts_with_non_starter(trie_value) {
680 combining_start = i;
681 }
682 }
683 (starter, combining_start)
684 }
685 }
686
687 fn push_decomposition32(
688 &mut self,
689 offset: usize,
690 len: usize,
691 only_non_starters_in_trail: bool,
692 slice32: &ZeroSlice<char>,
693 ) -> (char, usize) {
694 let (starter, tail) = slice32
695 .get_subslice(offset..offset + len)
696 .and_then(|slice| slice.split_first())
697 .unwrap_or_else(|| {
698 // GIGO case
699 debug_assert!(false);
700 (char::REPLACEMENT_CHARACTER, EMPTY_CHAR)
701 });
702 if only_non_starters_in_trail {
703 // All the rest are combining
704 self.buffer
705 .extend(tail.iter().map(CharacterAndClass::new_with_placeholder));
706 (starter, 0)
707 } else {
708 let mut i = 0;
709 let mut combining_start = 0;
710 for ch in tail.iter() {
711 let trie_value = self.trie.get(ch);
712 self.buffer.push(CharacterAndClass::new_with_trie_value(
713 CharacterAndTrieValue::new(ch, trie_value),
714 ));
715 i += 1;
716 // Half-width kana and iota subscript don't occur in the tails
717 // of these multicharacter decompositions.
718 if !decomposition_starts_with_non_starter(trie_value) {
719 combining_start = i;
720 }
721 }
722 (starter, combining_start)
723 }
724 }
725
726 #[inline(always)]
727 fn attach_trie_value(&self, c: char) -> CharacterAndTrieValue {
728 CharacterAndTrieValue::new(c, self.trie.get(c))
729 }
730
731 fn delegate_next_no_pending(&mut self) -> Option<CharacterAndTrieValue> {
732 debug_assert!(self.pending.is_none());
733 loop {
734 let c = self.delegate.next()?;
735
736 // TODO(#2384): Measure if this check is actually an optimization.
737 if u32::from(c) < self.decomposition_passthrough_bound {
738 return Some(CharacterAndTrieValue::new(c, 0));
739 }
740
741 let trie_val = self.trie.get(c);
742 // TODO: Can we do something better about the cost of this branch in the
743 // non-UTS 46 case?
744 if trie_val == IGNORABLE_MARKER {
745 match self.ignorable_behavior {
746 IgnorableBehavior::Unsupported => {
747 debug_assert!(false);
748 }
749 IgnorableBehavior::ReplacementCharacter => {
750 return Some(CharacterAndTrieValue::new(
751 c,
752 u32::from(char::REPLACEMENT_CHARACTER) | NON_ROUND_TRIP_MARKER,
753 ));
754 }
755 IgnorableBehavior::Ignored => {
756 // Else ignore this character by reading the next one from the delegate.
757 continue;
758 }
759 }
760 }
761 return Some(CharacterAndTrieValue::new(c, trie_val));
762 }
763 }
764
765 fn delegate_next(&mut self) -> Option<CharacterAndTrieValue> {
766 if let Some(pending) = self.pending.take() {
767 // Only happens as part of `Composition` and as part of
768 // the contiguous-buffer methods of `DecomposingNormalizer`.
769 // I.e. does not happen as part of standalone iterator
770 // usage of `Decomposition`.
771 Some(pending)
772 } else {
773 self.delegate_next_no_pending()
774 }
775 }
776
777 fn decomposing_next(&mut self, c_and_trie_val: CharacterAndTrieValue) -> char {
778 let (starter, combining_start) = {
779 let c = c_and_trie_val.character;
780 // See trie-value-format.md
781 let decomposition = c_and_trie_val.trie_val;
782 // The REPLACEMENT CHARACTER has `NON_ROUND_TRIP_MARKER` set,
783 // and that flag needs to be ignored here.
784 if (decomposition & !(BACKWARD_COMBINING_MARKER | NON_ROUND_TRIP_MARKER)) == 0 {
785 // The character is its own decomposition
786 (c, 0)
787 } else {
788 let high_zeros = (decomposition & HIGH_ZEROS_MASK) == 0;
789 let low_zeros = (decomposition & LOW_ZEROS_MASK) == 0;
790 if !high_zeros && !low_zeros {
791 // Decomposition into two BMP characters: starter and non-starter
792 let starter = char_from_u32(decomposition & 0x7FFF);
793 let combining = char_from_u32((decomposition >> 15) & 0x7FFF);
794 self.buffer
795 .push(CharacterAndClass::new_with_placeholder(combining));
796 (starter, 0)
797 } else if high_zeros {
798 // Do the check by looking at `c` instead of looking at a marker
799 // in `singleton` below, because if we looked at the trie value,
800 // we'd still have to check that `c` is in the Hangul syllable
801 // range in order for the subsequent interpretations as `char`
802 // to be safe.
803 // Alternatively, `FDFA_MARKER` and the Hangul marker could
804 // be unified. That would add a branch for Hangul and remove
805 // a branch from singleton decompositions. It seems more
806 // important to favor Hangul syllables than singleton
807 // decompositions.
808 // Note that it would be valid to hoist this Hangul check
809 // one or even two steps earlier in this check hierarchy.
810 // Right now, it's assumed the kind of decompositions into
811 // BMP starter and non-starter, which occur in many languages,
812 // should be checked before Hangul syllables, which are about
813 // one language specifically. Hopefully, we get some
814 // instruction-level parallelism out of the disjointness of
815 // operations on `c` and `decomposition`.
816 let hangul_offset = u32::from(c).wrapping_sub(HANGUL_S_BASE); // SIndex in the spec
817 if hangul_offset < HANGUL_S_COUNT {
818 debug_assert_eq!(decomposition, 1);
819 // Hangul syllable
820 // The math here comes from page 144 of Unicode 14.0
821 let l = hangul_offset / HANGUL_N_COUNT;
822 let v = (hangul_offset % HANGUL_N_COUNT) / HANGUL_T_COUNT;
823 let t = hangul_offset % HANGUL_T_COUNT;
824
825 // The unsafe blocks here are OK, because the values stay
826 // within the Hangul jamo block and, therefore, the scalar
827 // value range by construction.
828 self.buffer.push(CharacterAndClass::new_starter(unsafe {
829 char::from_u32_unchecked(HANGUL_V_BASE + v)
830 }));
831 let first = unsafe { char::from_u32_unchecked(HANGUL_L_BASE + l) };
832 if t != 0 {
833 self.buffer.push(CharacterAndClass::new_starter(unsafe {
834 char::from_u32_unchecked(HANGUL_T_BASE + t)
835 }));
836 (first, 2)
837 } else {
838 (first, 1)
839 }
840 } else {
841 let singleton = decomposition as u16;
842 if singleton != FDFA_MARKER {
843 // Decomposition into one BMP character
844 let starter = char_from_u16(singleton);
845 (starter, 0)
846 } else {
847 // Special case for the NFKD form of U+FDFA.
848 self.buffer.extend(FDFA_NFKD.map(|u| {
849 // SAFETY: `FDFA_NFKD` is known not to contain
850 // surrogates.
851 CharacterAndClass::new_starter(unsafe {
852 char::from_u32_unchecked(u32::from(u))
853 })
854 }));
855 ('\u{0635}', 17)
856 }
857 }
858 } else {
859 debug_assert!(low_zeros);
860 // Only 12 of 14 bits used as of Unicode 16.
861 let offset = (((decomposition & !(0b11 << 30)) >> 16) as usize) - 1;
862 // Only 3 of 4 bits used as of Unicode 16.
863 let len_bits = decomposition & 0b1111;
864 let only_non_starters_in_trail = (decomposition & 0b10000) != 0;
865 if offset < self.scalars16.len() {
866 self.push_decomposition16(
867 offset,
868 (len_bits + 2) as usize,
869 only_non_starters_in_trail,
870 self.scalars16,
871 )
872 } else if offset < self.scalars16.len() + self.scalars24.len() {
873 self.push_decomposition32(
874 offset - self.scalars16.len(),
875 (len_bits + 1) as usize,
876 only_non_starters_in_trail,
877 self.scalars24,
878 )
879 } else if offset
880 < self.scalars16.len()
881 + self.scalars24.len()
882 + self.supplementary_scalars16.len()
883 {
884 self.push_decomposition16(
885 offset - (self.scalars16.len() + self.scalars24.len()),
886 (len_bits + 2) as usize,
887 only_non_starters_in_trail,
888 self.supplementary_scalars16,
889 )
890 } else {
891 self.push_decomposition32(
892 offset
893 - (self.scalars16.len()
894 + self.scalars24.len()
895 + self.supplementary_scalars16.len()),
896 (len_bits + 1) as usize,
897 only_non_starters_in_trail,
898 self.supplementary_scalars24,
899 )
900 }
901 }
902 }
903 };
904 // Either we're inside `Composition` or `self.pending.is_none()`.
905
906 self.gather_and_sort_combining(combining_start);
907 starter
908 }
909
910 fn gather_and_sort_combining(&mut self, combining_start: usize) {
911 // Not a `for` loop to avoid holding a mutable reference to `self` across
912 // the loop body.
913 while let Some(ch_and_trie_val) = self.delegate_next() {
914 if !trie_value_has_ccc(ch_and_trie_val.trie_val) {
915 self.pending = Some(ch_and_trie_val);
916 break;
917 } else if !trie_value_indicates_special_non_starter_decomposition(
918 ch_and_trie_val.trie_val,
919 ) {
920 self.buffer
921 .push(CharacterAndClass::new_with_trie_value(ch_and_trie_val));
922 } else {
923 // The Tibetan special cases are starters that decompose into non-starters.
924 let mapped = match ch_and_trie_val.character {
925 '\u{0340}' => {
926 // COMBINING GRAVE TONE MARK
927 CharacterAndClass::new('\u{0300}', CanonicalCombiningClass::Above)
928 }
929 '\u{0341}' => {
930 // COMBINING ACUTE TONE MARK
931 CharacterAndClass::new('\u{0301}', CanonicalCombiningClass::Above)
932 }
933 '\u{0343}' => {
934 // COMBINING GREEK KORONIS
935 CharacterAndClass::new('\u{0313}', CanonicalCombiningClass::Above)
936 }
937 '\u{0344}' => {
938 // COMBINING GREEK DIALYTIKA TONOS
939 self.buffer.push(CharacterAndClass::new(
940 '\u{0308}',
941 CanonicalCombiningClass::Above,
942 ));
943 CharacterAndClass::new('\u{0301}', CanonicalCombiningClass::Above)
944 }
945 '\u{0F73}' => {
946 // TIBETAN VOWEL SIGN II
947 self.buffer.push(CharacterAndClass::new(
948 '\u{0F71}',
949 CanonicalCombiningClass(129),
950 ));
951 CharacterAndClass::new('\u{0F72}', CanonicalCombiningClass(130))
952 }
953 '\u{0F75}' => {
954 // TIBETAN VOWEL SIGN UU
955 self.buffer.push(CharacterAndClass::new(
956 '\u{0F71}',
957 CanonicalCombiningClass(129),
958 ));
959 CharacterAndClass::new('\u{0F74}', CanonicalCombiningClass(132))
960 }
961 '\u{0F81}' => {
962 // TIBETAN VOWEL SIGN REVERSED II
963 self.buffer.push(CharacterAndClass::new(
964 '\u{0F71}',
965 CanonicalCombiningClass(129),
966 ));
967 CharacterAndClass::new('\u{0F80}', CanonicalCombiningClass(130))
968 }
969 '\u{FF9E}' => {
970 // HALFWIDTH KATAKANA VOICED SOUND MARK
971 CharacterAndClass::new('\u{3099}', CanonicalCombiningClass::KanaVoicing)
972 }
973 '\u{FF9F}' => {
974 // HALFWIDTH KATAKANA VOICED SOUND MARK
975 CharacterAndClass::new('\u{309A}', CanonicalCombiningClass::KanaVoicing)
976 }
977 _ => {
978 // GIGO case
979 debug_assert!(false);
980 CharacterAndClass::new_with_placeholder(char::REPLACEMENT_CHARACTER)
981 }
982 };
983 self.buffer.push(mapped);
984 }
985 }
986 // Slicing succeeds by construction; we've always ensured that `combining_start`
987 // is in permissible range.
988 #[expect(clippy::indexing_slicing)]
989 sort_slice_by_ccc(&mut self.buffer[combining_start..], self.trie);
990 }
991}
992
993impl<I> Iterator for Decomposition<'_, I>
994where
995 I: Iterator<Item = char>,
996{
997 type Item = char;
998
999 fn next(&mut self) -> Option<char> {
1000 if let Some(ret) = self.buffer.get(self.buffer_pos).map(|c| c.character()) {
1001 self.buffer_pos += 1;
1002 if self.buffer_pos == self.buffer.len() {
1003 self.buffer.clear();
1004 self.buffer_pos = 0;
1005 }
1006 return Some(ret);
1007 }
1008 debug_assert_eq!(self.buffer_pos, 0);
1009 let c_and_trie_val = self.pending.take()?;
1010 Some(self.decomposing_next(c_and_trie_val))
1011 }
1012}
1013
1014/// An iterator adaptor that turns an `Iterator` over `char` into
1015/// a lazily-decomposed and then canonically composed `char` sequence.
1016#[derive(Debug)]
1017pub struct Composition<'data, I>
1018where
1019 I: Iterator<Item = char>,
1020{
1021 /// The decomposing part of the normalizer than operates before
1022 /// the canonical composition is performed on its output.
1023 decomposition: Decomposition<'data, I>,
1024 /// Non-Hangul canonical composition data.
1025 canonical_compositions: Char16Trie<'data>,
1026 /// To make `next()` yield in cases where there's a non-composing
1027 /// starter in the decomposition buffer, we put it here to let it
1028 /// wait for the next `next()` call (or a jump forward within the
1029 /// `next()` call).
1030 unprocessed_starter: Option<char>,
1031 /// The lowest character for which any one of the following does
1032 /// not hold:
1033 /// 1. Roundtrips via decomposition and recomposition.
1034 /// 2. Decomposition starts with a non-starter
1035 /// 3. Is not a backward-combining starter
1036 composition_passthrough_bound: u32,
1037}
1038
1039impl<'data, I> Composition<'data, I>
1040where
1041 I: Iterator<Item = char>,
1042{
1043 fn new(
1044 decomposition: Decomposition<'data, I>,
1045 canonical_compositions: Char16Trie<'data>,
1046 composition_passthrough_bound: u16,
1047 ) -> Self {
1048 Self {
1049 decomposition,
1050 canonical_compositions,
1051 unprocessed_starter: None,
1052 composition_passthrough_bound: u32::from(composition_passthrough_bound),
1053 }
1054 }
1055
1056 /// Performs canonical composition (including Hangul) on a pair of
1057 /// characters or returns `None` if these characters don't compose.
1058 /// Composition exclusions are taken into account.
1059 #[inline(always)]
1060 pub fn compose(&self, starter: char, second: char) -> Option<char> {
1061 compose(&self.canonical_compositions, starter, second)
1062 }
1063
1064 /// Performs (non-Hangul) canonical composition on a pair of characters
1065 /// or returns `None` if these characters don't compose. Composition
1066 /// exclusions are taken into account.
1067 #[inline(always)]
1068 fn compose_non_hangul(&self, starter: char, second: char) -> Option<char> {
1069 compose_non_hangul(&self.canonical_compositions, starter, second)
1070 }
1071}
1072
1073impl<I> Iterator for Composition<'_, I>
1074where
1075 I: Iterator<Item = char>,
1076{
1077 type Item = char;
1078
1079 #[inline]
1080 fn next(&mut self) -> Option<char> {
1081 let mut undecomposed_starter = CharacterAndTrieValue::new('\u{0}', 0); // The compiler can't figure out that this gets overwritten before use.
1082 if self.unprocessed_starter.is_none() {
1083 // The loop is only broken out of as goto forward
1084 #[expect(clippy::never_loop)]
1085 loop {
1086 if let Some((character, ccc)) = self
1087 .decomposition
1088 .buffer
1089 .get(self.decomposition.buffer_pos)
1090 .map(|c| c.character_and_ccc())
1091 {
1092 self.decomposition.buffer_pos += 1;
1093 if self.decomposition.buffer_pos == self.decomposition.buffer.len() {
1094 self.decomposition.buffer.clear();
1095 self.decomposition.buffer_pos = 0;
1096 }
1097 if ccc == CanonicalCombiningClass::NotReordered {
1098 // Previous decomposition contains a starter. This must
1099 // now become the `unprocessed_starter` for it to have
1100 // a chance to compose with the upcoming characters.
1101 //
1102 // E.g. parenthesized Hangul in NFKC comes through here,
1103 // but suitable composition exclusion could exercise this
1104 // in NFC.
1105 self.unprocessed_starter = Some(character);
1106 break; // We already have a starter, so skip taking one from `pending`.
1107 }
1108 return Some(character);
1109 }
1110 debug_assert_eq!(self.decomposition.buffer_pos, 0);
1111 undecomposed_starter = self.decomposition.pending.take()?;
1112 if u32::from(undecomposed_starter.character) < self.composition_passthrough_bound
1113 || undecomposed_starter.potential_passthrough()
1114 {
1115 // TODO(#2385): In the NFC case (moot for NFKC and UTS46), if the upcoming
1116 // character is not below `decomposition_passthrough_bound` but is
1117 // below `composition_passthrough_bound`, we read from the trie
1118 // unnecessarily.
1119 if let Some(upcoming) = self.decomposition.delegate_next_no_pending() {
1120 let cannot_combine_backwards = u32::from(upcoming.character)
1121 < self.composition_passthrough_bound
1122 || !upcoming.can_combine_backwards();
1123 self.decomposition.pending = Some(upcoming);
1124 if cannot_combine_backwards {
1125 // Fast-track succeeded!
1126 return Some(undecomposed_starter.character);
1127 }
1128 } else {
1129 // End of stream
1130 return Some(undecomposed_starter.character);
1131 }
1132 }
1133 break; // Not actually looping
1134 }
1135 }
1136 let mut starter = '\u{0}'; // The compiler can't figure out this gets overwritten before use.
1137
1138 // The point of having this boolean is to have only one call site to
1139 // `self.decomposition.decomposing_next`, which is hopefully beneficial for
1140 // code size under inlining.
1141 let mut attempt_composition = false;
1142 loop {
1143 if let Some(unprocessed) = self.unprocessed_starter.take() {
1144 debug_assert_eq!(undecomposed_starter, CharacterAndTrieValue::new('\u{0}', 0));
1145 debug_assert_eq!(starter, '\u{0}');
1146 starter = unprocessed;
1147 } else {
1148 debug_assert_eq!(self.decomposition.buffer_pos, 0);
1149 let next_starter = self.decomposition.decomposing_next(undecomposed_starter);
1150 if !attempt_composition {
1151 starter = next_starter;
1152 } else if let Some(composed) = self.compose(starter, next_starter) {
1153 starter = composed;
1154 } else {
1155 // This is our yield point. We'll pick this up above in the
1156 // next call to `next()`.
1157 self.unprocessed_starter = Some(next_starter);
1158 return Some(starter);
1159 }
1160 }
1161 // We first loop by index to avoid moving the contents of `buffer`, but
1162 // if there's a discontiguous match, we'll start modifying `buffer` instead.
1163 loop {
1164 let (character, ccc) = if let Some((character, ccc)) = self
1165 .decomposition
1166 .buffer
1167 .get(self.decomposition.buffer_pos)
1168 .map(|c| c.character_and_ccc())
1169 {
1170 (character, ccc)
1171 } else {
1172 self.decomposition.buffer.clear();
1173 self.decomposition.buffer_pos = 0;
1174 break;
1175 };
1176 if let Some(composed) = self.compose(starter, character) {
1177 starter = composed;
1178 self.decomposition.buffer_pos += 1;
1179 continue;
1180 }
1181 let mut most_recent_skipped_ccc = ccc;
1182 {
1183 let _ = self
1184 .decomposition
1185 .buffer
1186 .drain(0..self.decomposition.buffer_pos);
1187 }
1188 self.decomposition.buffer_pos = 0;
1189 if most_recent_skipped_ccc == CanonicalCombiningClass::NotReordered {
1190 // We failed to compose a starter. Discontiguous match not allowed.
1191 // We leave the starter in `buffer` for `next()` to find.
1192 return Some(starter);
1193 }
1194 let mut i = 1; // We have skipped one non-starter.
1195 while let Some((character, ccc)) = self
1196 .decomposition
1197 .buffer
1198 .get(i)
1199 .map(|c| c.character_and_ccc())
1200 {
1201 if ccc == CanonicalCombiningClass::NotReordered {
1202 // Discontiguous match not allowed.
1203 return Some(starter);
1204 }
1205 debug_assert!(ccc >= most_recent_skipped_ccc);
1206 if ccc != most_recent_skipped_ccc {
1207 // Using the non-Hangul version as a micro-optimization, since
1208 // we already rejected the case where `second` is a starter
1209 // above, and conjoining jamo are starters.
1210 if let Some(composed) = self.compose_non_hangul(starter, character) {
1211 self.decomposition.buffer.remove(i);
1212 starter = composed;
1213 continue;
1214 }
1215 }
1216 most_recent_skipped_ccc = ccc;
1217 i += 1;
1218 }
1219 break;
1220 }
1221
1222 debug_assert_eq!(self.decomposition.buffer_pos, 0);
1223
1224 if !self.decomposition.buffer.is_empty() {
1225 return Some(starter);
1226 }
1227 // Now we need to check if composition with an upcoming starter is possible.
1228 if let Some(pending) = self.decomposition.pending.take() {
1229 // We know that `pending_starter` decomposes to start with a starter.
1230 // Otherwise, it would have been moved to `self.decomposition.buffer`
1231 // by `self.decomposing_next()`. We do this set lookup here in order
1232 // to get an opportunity to go back to the fast track.
1233 // Note that this check has to happen _after_ checking that `pending`
1234 // holds a character, because this flag isn't defined to be meaningful
1235 // when `pending` isn't holding a character.
1236 if u32::from(pending.character) < self.composition_passthrough_bound
1237 || !pending.can_combine_backwards()
1238 {
1239 // Won't combine backwards anyway.
1240 self.decomposition.pending = Some(pending);
1241 return Some(starter);
1242 }
1243 // Consume what we peeked.
1244 undecomposed_starter = pending;
1245 // The following line is OK, because we're about to loop back
1246 // to `self.decomposition.decomposing_next(c);`, which will
1247 // restore the between-`next()`-calls invariant of `pending`
1248 // before this function returns.
1249 attempt_composition = true;
1250 continue;
1251 }
1252 // End of input
1253 return Some(starter);
1254 }
1255 }
1256}
1257
1258macro_rules! composing_normalize_to {
1259 ($(#[$meta:meta])*,
1260 $normalize_to:ident,
1261 $write:path,
1262 $slice:ty,
1263 $prolog:block,
1264 $always_valid_utf:literal,
1265 $as_slice:ident,
1266 $fast:block,
1267 $text:ident,
1268 $sink:ident,
1269 $composition:ident,
1270 $composition_passthrough_bound:ident,
1271 $undecomposed_starter:ident,
1272 $pending_slice:ident,
1273 $len_utf:ident,
1274 ) => {
1275 $(#[$meta])*
1276 pub fn $normalize_to<W: $write + ?Sized>(
1277 &self,
1278 $text: $slice,
1279 $sink: &mut W,
1280 ) -> core::fmt::Result {
1281 $prolog
1282 let mut $composition = self.normalize_iter($text.chars());
1283 debug_assert_eq!($composition.decomposition.ignorable_behavior, IgnorableBehavior::Unsupported);
1284 for cc in $composition.decomposition.buffer.drain(..) {
1285 $sink.write_char(cc.character())?;
1286 }
1287
1288 // Try to get the compiler to hoist the bound to a register.
1289 let $composition_passthrough_bound = $composition.composition_passthrough_bound;
1290 'outer: loop {
1291 debug_assert_eq!($composition.decomposition.buffer_pos, 0);
1292 let mut $undecomposed_starter =
1293 if let Some(pending) = $composition.decomposition.pending.take() {
1294 pending
1295 } else {
1296 return Ok(());
1297 };
1298 if u32::from($undecomposed_starter.character) < $composition_passthrough_bound ||
1299 $undecomposed_starter.potential_passthrough()
1300 {
1301 // We don't know if a `REPLACEMENT_CHARACTER` occurred in the slice or
1302 // was returned in response to an error by the iterator. Assume the
1303 // latter for correctness even though it pessimizes the former.
1304 if $always_valid_utf || $undecomposed_starter.character != char::REPLACEMENT_CHARACTER {
1305 let $pending_slice = &$text[$text.len() - $composition.decomposition.delegate.$as_slice().len() - $undecomposed_starter.character.$len_utf()..];
1306 // The `$fast` block must either:
1307 // 1. Return due to reaching EOF
1308 // 2. Leave a starter with its trie value in `$undecomposed_starter`
1309 // and, if there is still more input, leave the next character
1310 // and its trie value in `$composition.decomposition.pending`.
1311 $fast
1312 }
1313 }
1314 // Fast track above, full algorithm below
1315 let mut starter = $composition
1316 .decomposition
1317 .decomposing_next($undecomposed_starter);
1318 'bufferloop: loop {
1319 // We first loop by index to avoid moving the contents of `buffer`, but
1320 // if there's a discontiguous match, we'll start modifying `buffer` instead.
1321 loop {
1322 let (character, ccc) = if let Some((character, ccc)) = $composition
1323 .decomposition
1324 .buffer
1325 .get($composition.decomposition.buffer_pos)
1326 .map(|c| c.character_and_ccc())
1327 {
1328 (character, ccc)
1329 } else {
1330 $composition.decomposition.buffer.clear();
1331 $composition.decomposition.buffer_pos = 0;
1332 break;
1333 };
1334 if let Some(composed) = $composition.compose(starter, character) {
1335 starter = composed;
1336 $composition.decomposition.buffer_pos += 1;
1337 continue;
1338 }
1339 let mut most_recent_skipped_ccc = ccc;
1340 if most_recent_skipped_ccc == CanonicalCombiningClass::NotReordered {
1341 // We failed to compose a starter. Discontiguous match not allowed.
1342 // Write the current `starter` we've been composing, make the unmatched
1343 // starter in the buffer the new `starter` (we know it's been decomposed)
1344 // and process the rest of the buffer with that as the starter.
1345 $sink.write_char(starter)?;
1346 starter = character;
1347 $composition.decomposition.buffer_pos += 1;
1348 continue 'bufferloop;
1349 } else {
1350 {
1351 let _ = $composition
1352 .decomposition
1353 .buffer
1354 .drain(0..$composition.decomposition.buffer_pos);
1355 }
1356 $composition.decomposition.buffer_pos = 0;
1357 }
1358 let mut i = 1; // We have skipped one non-starter.
1359 while let Some((character, ccc)) = $composition
1360 .decomposition
1361 .buffer
1362 .get(i)
1363 .map(|c| c.character_and_ccc())
1364 {
1365 if ccc == CanonicalCombiningClass::NotReordered {
1366 // Discontiguous match not allowed.
1367 $sink.write_char(starter)?;
1368 for cc in $composition.decomposition.buffer.drain(..i) {
1369 $sink.write_char(cc.character())?;
1370 }
1371 starter = character;
1372 {
1373 let removed = $composition.decomposition.buffer.remove(0);
1374 debug_assert_eq!(starter, removed.character());
1375 }
1376 debug_assert_eq!($composition.decomposition.buffer_pos, 0);
1377 continue 'bufferloop;
1378 }
1379 debug_assert!(ccc >= most_recent_skipped_ccc);
1380 if ccc != most_recent_skipped_ccc {
1381 // Using the non-Hangul version as a micro-optimization, since
1382 // we already rejected the case where `second` is a starter
1383 // above, and conjoining jamo are starters.
1384 if let Some(composed) =
1385 $composition.compose_non_hangul(starter, character)
1386 {
1387 $composition.decomposition.buffer.remove(i);
1388 starter = composed;
1389 continue;
1390 }
1391 }
1392 most_recent_skipped_ccc = ccc;
1393 i += 1;
1394 }
1395 break;
1396 }
1397 debug_assert_eq!($composition.decomposition.buffer_pos, 0);
1398
1399 if !$composition.decomposition.buffer.is_empty() {
1400 $sink.write_char(starter)?;
1401 for cc in $composition.decomposition.buffer.drain(..) {
1402 $sink.write_char(cc.character())?;
1403 }
1404 // We had non-empty buffer, so can't compose with upcoming.
1405 continue 'outer;
1406 }
1407 // Now we need to check if composition with an upcoming starter is possible.
1408 if $composition.decomposition.pending.is_some() {
1409 // We know that `pending_starter` decomposes to start with a starter.
1410 // Otherwise, it would have been moved to `composition.decomposition.buffer`
1411 // by `composition.decomposing_next()`. We do this set lookup here in order
1412 // to get an opportunity to go back to the fast track.
1413 // Note that this check has to happen _after_ checking that `pending`
1414 // holds a character, because this flag isn't defined to be meaningful
1415 // when `pending` isn't holding a character.
1416 let pending = $composition.decomposition.pending.as_ref().unwrap();
1417 if u32::from(pending.character) < $composition.composition_passthrough_bound
1418 || !pending.can_combine_backwards()
1419 {
1420 // Won't combine backwards anyway.
1421 $sink.write_char(starter)?;
1422 continue 'outer;
1423 }
1424 let pending_starter = $composition.decomposition.pending.take().unwrap();
1425 let decomposed = $composition.decomposition.decomposing_next(pending_starter);
1426 if let Some(composed) = $composition.compose(starter, decomposed) {
1427 starter = composed;
1428 } else {
1429 $sink.write_char(starter)?;
1430 starter = decomposed;
1431 }
1432 continue 'bufferloop;
1433 }
1434 // End of input
1435 $sink.write_char(starter)?;
1436 return Ok(());
1437 } // 'bufferloop
1438 }
1439 }
1440 };
1441}
1442
1443macro_rules! decomposing_normalize_to {
1444 ($(#[$meta:meta])*,
1445 $normalize_to:ident,
1446 $write:path,
1447 $slice:ty,
1448 $prolog:block,
1449 $as_slice:ident,
1450 $fast:block,
1451 $text:ident,
1452 $sink:ident,
1453 $decomposition:ident,
1454 $decomposition_passthrough_bound:ident,
1455 $undecomposed_starter:ident,
1456 $pending_slice:ident,
1457 $outer:lifetime, // loop labels use lifetime tokens
1458 ) => {
1459 $(#[$meta])*
1460 pub fn $normalize_to<W: $write + ?Sized>(
1461 &self,
1462 $text: $slice,
1463 $sink: &mut W,
1464 ) -> core::fmt::Result {
1465 $prolog
1466
1467 let mut $decomposition = self.normalize_iter($text.chars());
1468 debug_assert_eq!($decomposition.ignorable_behavior, IgnorableBehavior::Unsupported);
1469
1470 // Try to get the compiler to hoist the bound to a register.
1471 let $decomposition_passthrough_bound = $decomposition.decomposition_passthrough_bound;
1472 $outer: loop {
1473 for cc in $decomposition.buffer.drain(..) {
1474 $sink.write_char(cc.character())?;
1475 }
1476 debug_assert_eq!($decomposition.buffer_pos, 0);
1477 let mut $undecomposed_starter = if let Some(pending) = $decomposition.pending.take() {
1478 pending
1479 } else {
1480 return Ok(());
1481 };
1482 if $undecomposed_starter.starter_and_decomposes_to_self() {
1483 // Don't bother including `undecomposed_starter` in a contiguous buffer
1484 // write: Just write it right away:
1485 $sink.write_char($undecomposed_starter.character)?;
1486
1487 let $pending_slice = $decomposition.delegate.$as_slice();
1488 $fast
1489 }
1490 let starter = $decomposition.decomposing_next($undecomposed_starter);
1491 $sink.write_char(starter)?;
1492 }
1493 }
1494 };
1495}
1496
1497macro_rules! normalizer_methods {
1498 () => {
1499 /// Normalize a string slice into a `Cow<'a, str>`.
1500 pub fn normalize<'a>(&self, text: &'a str) -> Cow<'a, str> {
1501 let (head, tail) = self.split_normalized(text);
1502 if tail.is_empty() {
1503 return Cow::Borrowed(head);
1504 }
1505 let mut ret = String::new();
1506 ret.reserve(text.len());
1507 ret.push_str(head);
1508 let _ = self.normalize_to(tail, &mut ret);
1509 Cow::Owned(ret)
1510 }
1511
1512 /// Split a string slice into maximum normalized prefix and unnormalized suffix
1513 /// such that the concatenation of the prefix and the normalization of the suffix
1514 /// is the normalization of the whole input.
1515 pub fn split_normalized<'a>(&self, text: &'a str) -> (&'a str, &'a str) {
1516 let up_to = self.is_normalized_up_to(text);
1517 text.split_at_checked(up_to).unwrap_or_else(|| {
1518 // Internal bug, not even GIGO, never supposed to happen
1519 debug_assert!(false);
1520 ("", text)
1521 })
1522 }
1523
1524 /// Return the index a string slice is normalized up to.
1525 fn is_normalized_up_to(&self, text: &str) -> usize {
1526 let mut sink = IsNormalizedSinkStr::new(text);
1527 let _ = self.normalize_to(text, &mut sink);
1528 text.len() - sink.remaining_len()
1529 }
1530
1531 /// Check whether a string slice is normalized.
1532 pub fn is_normalized(&self, text: &str) -> bool {
1533 self.is_normalized_up_to(text) == text.len()
1534 }
1535
1536 /// Normalize a slice of potentially-invalid UTF-16 into a `Cow<'a, [u16]>`.
1537 ///
1538 /// Unpaired surrogates are mapped to the REPLACEMENT CHARACTER
1539 /// before normalizing.
1540 ///
1541 /// ✨ *Enabled with the `utf16_iter` Cargo feature.*
1542 #[cfg(feature = "utf16_iter")]
1543 pub fn normalize_utf16<'a>(&self, text: &'a [u16]) -> Cow<'a, [u16]> {
1544 let (head, tail) = self.split_normalized_utf16(text);
1545 if tail.is_empty() {
1546 return Cow::Borrowed(head);
1547 }
1548 let mut ret = alloc::vec::Vec::with_capacity(text.len());
1549 ret.extend_from_slice(head);
1550 let _ = self.normalize_utf16_to(tail, &mut ret);
1551 Cow::Owned(ret)
1552 }
1553
1554 /// Split a slice of potentially-invalid UTF-16 into maximum normalized (and valid)
1555 /// prefix and unnormalized suffix such that the concatenation of the prefix and the
1556 /// normalization of the suffix is the normalization of the whole input.
1557 ///
1558 /// ✨ *Enabled with the `utf16_iter` Cargo feature.*
1559 #[cfg(feature = "utf16_iter")]
1560 pub fn split_normalized_utf16<'a>(&self, text: &'a [u16]) -> (&'a [u16], &'a [u16]) {
1561 let up_to = self.is_normalized_utf16_up_to(text);
1562 text.split_at_checked(up_to).unwrap_or_else(|| {
1563 // Internal bug, not even GIGO, never supposed to happen
1564 debug_assert!(false);
1565 (&[], text)
1566 })
1567 }
1568
1569 /// Return the index a slice of potentially-invalid UTF-16 is normalized up to.
1570 ///
1571 /// ✨ *Enabled with the `utf16_iter` Cargo feature.*
1572 #[cfg(feature = "utf16_iter")]
1573 fn is_normalized_utf16_up_to(&self, text: &[u16]) -> usize {
1574 let mut sink = IsNormalizedSinkUtf16::new(text);
1575 let _ = self.normalize_utf16_to(text, &mut sink);
1576 text.len() - sink.remaining_len()
1577 }
1578
1579 /// Checks whether a slice of potentially-invalid UTF-16 is normalized.
1580 ///
1581 /// Unpaired surrogates are treated as the REPLACEMENT CHARACTER.
1582 ///
1583 /// ✨ *Enabled with the `utf16_iter` Cargo feature.*
1584 #[cfg(feature = "utf16_iter")]
1585 pub fn is_normalized_utf16(&self, text: &[u16]) -> bool {
1586 self.is_normalized_utf16_up_to(text) == text.len()
1587 }
1588
1589 /// Normalize a slice of potentially-invalid UTF-8 into a `Cow<'a, str>`.
1590 ///
1591 /// Ill-formed byte sequences are mapped to the REPLACEMENT CHARACTER
1592 /// according to the WHATWG Encoding Standard.
1593 ///
1594 /// ✨ *Enabled with the `utf8_iter` Cargo feature.*
1595 #[cfg(feature = "utf8_iter")]
1596 pub fn normalize_utf8<'a>(&self, text: &'a [u8]) -> Cow<'a, str> {
1597 let (head, tail) = self.split_normalized_utf8(text);
1598 if tail.is_empty() {
1599 return Cow::Borrowed(head);
1600 }
1601 let mut ret = String::new();
1602 ret.reserve(text.len());
1603 ret.push_str(head);
1604 let _ = self.normalize_utf8_to(tail, &mut ret);
1605 Cow::Owned(ret)
1606 }
1607
1608 /// Split a slice of potentially-invalid UTF-8 into maximum normalized (and valid)
1609 /// prefix and unnormalized suffix such that the concatenation of the prefix and the
1610 /// normalization of the suffix is the normalization of the whole input.
1611 ///
1612 /// ✨ *Enabled with the `utf8_iter` Cargo feature.*
1613 #[cfg(feature = "utf8_iter")]
1614 pub fn split_normalized_utf8<'a>(&self, text: &'a [u8]) -> (&'a str, &'a [u8]) {
1615 let up_to = self.is_normalized_utf8_up_to(text);
1616 let (head, tail) = text.split_at_checked(up_to).unwrap_or_else(|| {
1617 // Internal bug, not even GIGO, never supposed to happen
1618 debug_assert!(false);
1619 (&[], text)
1620 });
1621 // SAFETY: The normalization check also checks for
1622 // UTF-8 well-formedness.
1623 (unsafe { str::from_utf8_unchecked(head) }, tail)
1624 }
1625
1626 /// Return the index a slice of potentially-invalid UTF-8 is normalized up to
1627 ///
1628 /// ✨ *Enabled with the `utf8_iter` Cargo feature.*
1629 #[cfg(feature = "utf8_iter")]
1630 fn is_normalized_utf8_up_to(&self, text: &[u8]) -> usize {
1631 let mut sink = IsNormalizedSinkUtf8::new(text);
1632 let _ = self.normalize_utf8_to(text, &mut sink);
1633 text.len() - sink.remaining_len()
1634 }
1635
1636 /// Check if a slice of potentially-invalid UTF-8 is normalized.
1637 ///
1638 /// Ill-formed byte sequences are mapped to the REPLACEMENT CHARACTER
1639 /// according to the WHATWG Encoding Standard before checking.
1640 ///
1641 /// ✨ *Enabled with the `utf8_iter` Cargo feature.*
1642 #[cfg(feature = "utf8_iter")]
1643 pub fn is_normalized_utf8(&self, text: &[u8]) -> bool {
1644 self.is_normalized_utf8_up_to(text) == text.len()
1645 }
1646 };
1647}
1648
1649/// Borrowed version of a normalizer for performing decomposing normalization.
1650#[derive(Debug)]
1651pub struct DecomposingNormalizerBorrowed<'a> {
1652 decompositions: &'a DecompositionData<'a>,
1653 tables: &'a DecompositionTables<'a>,
1654 supplementary_tables: Option<&'a DecompositionTables<'a>>,
1655 decomposition_passthrough_bound: u8, // never above 0xC0
1656 composition_passthrough_bound: u16, // never above 0x0300
1657}
1658
1659impl DecomposingNormalizerBorrowed<'static> {
1660 /// Cheaply converts a [`DecomposingNormalizerBorrowed<'static>`] into a [`DecomposingNormalizer`].
1661 ///
1662 /// Note: Due to branching and indirection, using [`DecomposingNormalizer`] might inhibit some
1663 /// compile-time optimizations that are possible with [`DecomposingNormalizerBorrowed`].
1664 pub const fn static_to_owned(self) -> DecomposingNormalizer {
1665 DecomposingNormalizer {
1666 decompositions: DataPayload::from_static_ref(self.decompositions),
1667 tables: DataPayload::from_static_ref(self.tables),
1668 supplementary_tables: if let Some(s) = self.supplementary_tables {
1669 // `map` not available in const context
1670 Some(DataPayload::from_static_ref(s))
1671 } else {
1672 None
1673 },
1674 decomposition_passthrough_bound: self.decomposition_passthrough_bound,
1675 composition_passthrough_bound: self.composition_passthrough_bound,
1676 }
1677 }
1678
1679 /// NFD constructor using compiled data.
1680 ///
1681 /// ✨ *Enabled with the `compiled_data` Cargo feature.*
1682 ///
1683 /// [📚 Help choosing a constructor](icu_provider::constructors)
1684 #[cfg(feature = "compiled_data")]
1685 pub const fn new_nfd() -> Self {
1686 const _: () = assert!(
1687 provider::Baked::SINGLETON_NORMALIZER_NFD_TABLES_V1
1688 .scalars16
1689 .const_len()
1690 + provider::Baked::SINGLETON_NORMALIZER_NFD_TABLES_V1
1691 .scalars24
1692 .const_len()
1693 <= 0xFFF,
1694 "future extension"
1695 );
1696
1697 DecomposingNormalizerBorrowed {
1698 decompositions: provider::Baked::SINGLETON_NORMALIZER_NFD_DATA_V1,
1699 tables: provider::Baked::SINGLETON_NORMALIZER_NFD_TABLES_V1,
1700 supplementary_tables: None,
1701 decomposition_passthrough_bound: 0xC0,
1702 composition_passthrough_bound: 0x0300,
1703 }
1704 }
1705
1706 /// NFKD constructor using compiled data.
1707 ///
1708 /// ✨ *Enabled with the `compiled_data` Cargo feature.*
1709 ///
1710 /// [📚 Help choosing a constructor](icu_provider::constructors)
1711 #[cfg(feature = "compiled_data")]
1712 pub const fn new_nfkd() -> Self {
1713 const _: () = assert!(
1714 provider::Baked::SINGLETON_NORMALIZER_NFD_TABLES_V1
1715 .scalars16
1716 .const_len()
1717 + provider::Baked::SINGLETON_NORMALIZER_NFD_TABLES_V1
1718 .scalars24
1719 .const_len()
1720 + provider::Baked::SINGLETON_NORMALIZER_NFKD_TABLES_V1
1721 .scalars16
1722 .const_len()
1723 + provider::Baked::SINGLETON_NORMALIZER_NFKD_TABLES_V1
1724 .scalars24
1725 .const_len()
1726 <= 0xFFF,
1727 "future extension"
1728 );
1729
1730 const _: () = assert!(
1731 provider::Baked::SINGLETON_NORMALIZER_NFKD_DATA_V1.passthrough_cap <= 0x0300,
1732 "invalid"
1733 );
1734
1735 let decomposition_capped =
1736 if provider::Baked::SINGLETON_NORMALIZER_NFKD_DATA_V1.passthrough_cap < 0xC0 {
1737 provider::Baked::SINGLETON_NORMALIZER_NFKD_DATA_V1.passthrough_cap
1738 } else {
1739 0xC0
1740 };
1741 let composition_capped =
1742 if provider::Baked::SINGLETON_NORMALIZER_NFKD_DATA_V1.passthrough_cap < 0x0300 {
1743 provider::Baked::SINGLETON_NORMALIZER_NFKD_DATA_V1.passthrough_cap
1744 } else {
1745 0x0300
1746 };
1747
1748 DecomposingNormalizerBorrowed {
1749 decompositions: provider::Baked::SINGLETON_NORMALIZER_NFKD_DATA_V1,
1750 tables: provider::Baked::SINGLETON_NORMALIZER_NFD_TABLES_V1,
1751 supplementary_tables: Some(provider::Baked::SINGLETON_NORMALIZER_NFKD_TABLES_V1),
1752 decomposition_passthrough_bound: decomposition_capped as u8,
1753 composition_passthrough_bound: composition_capped,
1754 }
1755 }
1756
1757 #[cfg(feature = "compiled_data")]
1758 pub(crate) const fn new_uts46_decomposed() -> Self {
1759 const _: () = assert!(
1760 provider::Baked::SINGLETON_NORMALIZER_NFD_TABLES_V1
1761 .scalars16
1762 .const_len()
1763 + provider::Baked::SINGLETON_NORMALIZER_NFD_TABLES_V1
1764 .scalars24
1765 .const_len()
1766 + provider::Baked::SINGLETON_NORMALIZER_NFKD_TABLES_V1
1767 .scalars16
1768 .const_len()
1769 + provider::Baked::SINGLETON_NORMALIZER_NFKD_TABLES_V1
1770 .scalars24
1771 .const_len()
1772 <= 0xFFF,
1773 "future extension"
1774 );
1775
1776 const _: () = assert!(
1777 provider::Baked::SINGLETON_NORMALIZER_UTS46_DATA_V1.passthrough_cap <= 0x0300,
1778 "invalid"
1779 );
1780
1781 let decomposition_capped =
1782 if provider::Baked::SINGLETON_NORMALIZER_UTS46_DATA_V1.passthrough_cap < 0xC0 {
1783 provider::Baked::SINGLETON_NORMALIZER_UTS46_DATA_V1.passthrough_cap
1784 } else {
1785 0xC0
1786 };
1787 let composition_capped =
1788 if provider::Baked::SINGLETON_NORMALIZER_UTS46_DATA_V1.passthrough_cap < 0x0300 {
1789 provider::Baked::SINGLETON_NORMALIZER_UTS46_DATA_V1.passthrough_cap
1790 } else {
1791 0x0300
1792 };
1793
1794 DecomposingNormalizerBorrowed {
1795 decompositions: provider::Baked::SINGLETON_NORMALIZER_UTS46_DATA_V1,
1796 tables: provider::Baked::SINGLETON_NORMALIZER_NFD_TABLES_V1,
1797 supplementary_tables: Some(provider::Baked::SINGLETON_NORMALIZER_NFKD_TABLES_V1),
1798 decomposition_passthrough_bound: decomposition_capped as u8,
1799 composition_passthrough_bound: composition_capped,
1800 }
1801 }
1802}
1803
1804impl<'data> DecomposingNormalizerBorrowed<'data> {
1805 /// NFD constructor using already-loaded data.
1806 ///
1807 /// This constructor is intended for use by collations.
1808 ///
1809 /// [📚 Help choosing a constructor](icu_provider::constructors)
1810 #[doc(hidden)]
1811 pub fn new_with_data(
1812 decompositions: &'data DecompositionData<'data>,
1813 tables: &'data DecompositionTables<'data>,
1814 ) -> Self {
1815 Self {
1816 decompositions,
1817 tables,
1818 supplementary_tables: None,
1819 decomposition_passthrough_bound: 0xC0,
1820 composition_passthrough_bound: 0x0300,
1821 }
1822 }
1823
1824 /// Wraps a delegate iterator into a decomposing iterator
1825 /// adapter by using the data already held by this normalizer.
1826 pub fn normalize_iter<I: Iterator<Item = char>>(&self, iter: I) -> Decomposition<'data, I> {
1827 Decomposition::new_with_supplements(
1828 iter,
1829 self.decompositions,
1830 self.tables,
1831 self.supplementary_tables,
1832 self.decomposition_passthrough_bound,
1833 IgnorableBehavior::Unsupported,
1834 )
1835 }
1836
1837 normalizer_methods!();
1838
1839 decomposing_normalize_to!(
1840 /// Normalize a string slice into a `Write` sink.
1841 ,
1842 normalize_to,
1843 core::fmt::Write,
1844 &str,
1845 {
1846 },
1847 as_str,
1848 {
1849 let decomposition_passthrough_byte_bound = if decomposition_passthrough_bound == 0xC0 {
1850 0xC3u8
1851 } else {
1852 decomposition_passthrough_bound.min(0x80) as u8
1853 };
1854 // The attribute belongs on an inner statement, but Rust doesn't allow it there.
1855 #[expect(clippy::unwrap_used)]
1856 'fast: loop {
1857 let mut code_unit_iter = decomposition.delegate.as_str().as_bytes().iter();
1858 'fastest: loop {
1859 if let Some(&upcoming_byte) = code_unit_iter.next() {
1860 if upcoming_byte < decomposition_passthrough_byte_bound {
1861 // Fast-track succeeded!
1862 continue 'fastest;
1863 }
1864 // This deliberately isn't panic-free, since the code pattern
1865 // that was OK for the composing counterpart regressed
1866 // English and French performance if done here, too.
1867 decomposition.delegate = pending_slice[pending_slice.len() - code_unit_iter.as_slice().len() - 1..].chars();
1868 break 'fastest;
1869 }
1870 // End of stream
1871 sink.write_str(pending_slice)?;
1872 return Ok(());
1873 }
1874
1875 // `unwrap()` OK, because the slice is valid UTF-8 and we know there
1876 // is an upcoming byte.
1877 let upcoming = decomposition.delegate.next().unwrap();
1878 let upcoming_with_trie_value = decomposition.attach_trie_value(upcoming);
1879 if upcoming_with_trie_value.starter_and_decomposes_to_self() {
1880 continue 'fast;
1881 }
1882 let consumed_so_far_slice = &pending_slice[..pending_slice.len()
1883 - decomposition.delegate.as_str().len()
1884 - upcoming.len_utf8()];
1885 sink.write_str(consumed_so_far_slice)?;
1886
1887 // Now let's figure out if we got a starter or a non-starter.
1888 if decomposition_starts_with_non_starter(
1889 upcoming_with_trie_value.trie_val,
1890 ) {
1891 // Let this trie value to be reprocessed in case it is
1892 // one of the rare decomposing ones.
1893 decomposition.pending = Some(upcoming_with_trie_value);
1894 decomposition.gather_and_sort_combining(0);
1895 continue 'outer;
1896 }
1897 undecomposed_starter = upcoming_with_trie_value;
1898 debug_assert!(decomposition.pending.is_none());
1899 break 'fast;
1900 }
1901 },
1902 text,
1903 sink,
1904 decomposition,
1905 decomposition_passthrough_bound,
1906 undecomposed_starter,
1907 pending_slice,
1908 'outer,
1909 );
1910
1911 decomposing_normalize_to!(
1912 /// Normalize a slice of potentially-invalid UTF-8 into a `Write` sink.
1913 ///
1914 /// Ill-formed byte sequences are mapped to the REPLACEMENT CHARACTER
1915 /// according to the WHATWG Encoding Standard.
1916 ///
1917 /// ✨ *Enabled with the `utf8_iter` Cargo feature.*
1918 #[cfg(feature = "utf8_iter")]
1919 ,
1920 normalize_utf8_to,
1921 core::fmt::Write,
1922 &[u8],
1923 {
1924 },
1925 as_slice,
1926 {
1927 let decomposition_passthrough_byte_bound = decomposition_passthrough_bound.min(0x80) as u8;
1928 'fast: loop {
1929 let mut code_unit_iter = decomposition.delegate.as_slice().iter();
1930 'fastest: loop {
1931 if let Some(&upcoming_byte) = code_unit_iter.next() {
1932 if upcoming_byte < decomposition_passthrough_byte_bound {
1933 // Fast-track succeeded!
1934 continue 'fastest;
1935 }
1936 break 'fastest;
1937 }
1938 // End of stream
1939 sink.write_str(unsafe { str::from_utf8_unchecked(pending_slice) })?;
1940 return Ok(());
1941 }
1942 #[expect(clippy::indexing_slicing)]
1943 {decomposition.delegate = pending_slice[pending_slice.len() - code_unit_iter.as_slice().len() - 1..].chars();}
1944
1945 // `unwrap()` OK, because the slice is valid UTF-8 and we know there
1946 // is an upcoming byte.
1947 #[expect(clippy::unwrap_used)]
1948 let upcoming = decomposition.delegate.next().unwrap();
1949 let upcoming_with_trie_value = decomposition.attach_trie_value(upcoming);
1950 if upcoming_with_trie_value.starter_and_decomposes_to_self_except_replacement() {
1951 // Note: The trie value of the REPLACEMENT CHARACTER is
1952 // intentionally formatted to fail the
1953 // `starter_and_decomposes_to_self` test even though it
1954 // really is a starter that decomposes to self. This
1955 // Allows moving the branch on REPLACEMENT CHARACTER
1956 // below this `continue`.
1957 continue 'fast;
1958 }
1959
1960 // TODO: Annotate as unlikely.
1961 if upcoming == char::REPLACEMENT_CHARACTER {
1962 // We might have an error, so fall out of the fast path.
1963
1964 // Since the U+FFFD might signify an error, we can't
1965 // assume `upcoming.len_utf8()` for the backoff length.
1966 #[expect(clippy::indexing_slicing)]
1967 let mut consumed_so_far = pending_slice[..pending_slice.len() - decomposition.delegate.as_slice().len()].chars();
1968 let back = consumed_so_far.next_back();
1969 debug_assert_eq!(back, Some(char::REPLACEMENT_CHARACTER));
1970 let consumed_so_far_slice = consumed_so_far.as_slice();
1971 sink.write_str(unsafe { str::from_utf8_unchecked(consumed_so_far_slice) } )?;
1972
1973 // We could call `gather_and_sort_combining` here and
1974 // `continue 'outer`, but this should be better for code
1975 // size.
1976 undecomposed_starter = upcoming_with_trie_value;
1977 debug_assert!(decomposition.pending.is_none());
1978 break 'fast;
1979 }
1980
1981 #[expect(clippy::indexing_slicing)]
1982 let consumed_so_far_slice = &pending_slice[..pending_slice.len()
1983 - decomposition.delegate.as_slice().len()
1984 - upcoming.len_utf8()];
1985 sink.write_str(unsafe { str::from_utf8_unchecked(consumed_so_far_slice) } )?;
1986
1987 // Now let's figure out if we got a starter or a non-starter.
1988 if decomposition_starts_with_non_starter(
1989 upcoming_with_trie_value.trie_val,
1990 ) {
1991 // Let this trie value to be reprocessed in case it is
1992 // one of the rare decomposing ones.
1993 decomposition.pending = Some(upcoming_with_trie_value);
1994 decomposition.gather_and_sort_combining(0);
1995 continue 'outer;
1996 }
1997 undecomposed_starter = upcoming_with_trie_value;
1998 debug_assert!(decomposition.pending.is_none());
1999 break 'fast;
2000 }
2001 },
2002 text,
2003 sink,
2004 decomposition,
2005 decomposition_passthrough_bound,
2006 undecomposed_starter,
2007 pending_slice,
2008 'outer,
2009 );
2010
2011 decomposing_normalize_to!(
2012 /// Normalize a slice of potentially-invalid UTF-16 into a `Write16` sink.
2013 ///
2014 /// Unpaired surrogates are mapped to the REPLACEMENT CHARACTER
2015 /// before normalizing.
2016 ///
2017 /// ✨ *Enabled with the `utf16_iter` Cargo feature.*
2018 #[cfg(feature = "utf16_iter")]
2019 ,
2020 normalize_utf16_to,
2021 write16::Write16,
2022 &[u16],
2023 {
2024 sink.size_hint(text.len())?;
2025 },
2026 as_slice,
2027 {
2028 // This loop is only broken out of as goto forward and only as release-build recovery from
2029 // detecting an internal bug without panic. (In debug builds, internal bugs panic instead.)
2030 #[expect(clippy::never_loop)]
2031 'fastwrap: loop {
2032 // Commented out `code_unit_iter` and used `ptr` and `end` to
2033 // work around https://github.com/rust-lang/rust/issues/144684 .
2034 //
2035 // let mut code_unit_iter = decomposition.delegate.as_slice().iter();
2036 let delegate_as_slice = decomposition.delegate.as_slice();
2037 let mut ptr: *const u16 = delegate_as_slice.as_ptr();
2038 // SAFETY: materializing a pointer immediately past the end of an
2039 // allocation is OK.
2040 let end: *const u16 = unsafe { ptr.add(delegate_as_slice.len()) };
2041 'fast: loop {
2042 // if let Some(&upcoming_code_unit) = code_unit_iter.next() {
2043 if ptr != end {
2044 // SAFETY: We just checked that `ptr` has not reached `end`.
2045 // `ptr` always advances by one, and we always have a check
2046 // per advancement.
2047 let upcoming_code_unit = unsafe { *ptr };
2048 // SAFETY: Since `ptr` hadn't reached `end`, yet, advancing
2049 // by one points to the same allocation or to immediately
2050 // after, which is OK.
2051 ptr = unsafe { ptr.add(1) };
2052
2053 let mut upcoming32 = u32::from(upcoming_code_unit);
2054 // The performance of what logically is supposed to be this
2055 // branch is _incredibly_ brittle and what LLVM ends up doing
2056 // that affects the performance of what's logically about this
2057 // decision can swing to double/halve the throughput for Basic
2058 // Latin in ways that are completely unintuitive. Basically _any_
2059 // change to _any_ code that participates in how LLVM sees the
2060 // code around here can make the perf fall over. In seems that
2061 // manually annotating this branch as likely has worse effects
2062 // on non-Basic-Latin input that the case where LLVM just happens to
2063 // do the right thing.
2064 //
2065 // What happens with this branch may depend on what sink type
2066 // this code is monomorphized over.
2067 //
2068 // What a terrible sink of developer time!
2069 if upcoming32 < decomposition_passthrough_bound {
2070 continue 'fast;
2071 }
2072 // We might be doing a trie lookup by surrogate. Surrogates get
2073 // a decomposition to U+FFFD.
2074 let mut trie_value = decomposition.trie.get16(upcoming_code_unit);
2075 if starter_and_decomposes_to_self_impl(trie_value) {
2076 continue 'fast;
2077 }
2078 // We might now be looking at a surrogate.
2079 // The loop is only broken out of as goto forward
2080 #[expect(clippy::never_loop)]
2081 'surrogateloop: loop {
2082 // LLVM's optimizations are incredibly brittle for the code _above_,
2083 // and using `likely` _below_ without using it _above_ helps!
2084 // What a massive sink of developer time!
2085 // Seriously, the effect of these annotations is massively
2086 // unintuitive. Measure everything!
2087 // Notably, the `if likely(...)` formulation optimizes differently
2088 // than just putting `cold_path()` on the `else` path!
2089 let surrogate_base = upcoming32.wrapping_sub(0xD800);
2090 if likely(surrogate_base > (0xDFFF - 0xD800)) {
2091 // Not surrogate
2092 break 'surrogateloop;
2093 }
2094 if likely(surrogate_base <= (0xDBFF - 0xD800)) {
2095 // let iter_backup = code_unit_iter.clone();
2096 // if let Some(&low) = code_unit_iter.next() {
2097 if ptr != end {
2098 // SAFETY: We just checked that `ptr` has not reached `end`.
2099 // `ptr` always advances by one, and we always have a check
2100 // per advancement.
2101 let low = unsafe { *ptr };
2102 if likely(in_inclusive_range16(low, 0xDC00, 0xDFFF)) {
2103 // SAFETY: Since `ptr` hadn't reached `end`, yet, advancing
2104 // by one points to the same allocation or to immediately
2105 // after, which is OK.
2106 ptr = unsafe { ptr.add(1) };
2107
2108 upcoming32 = (upcoming32 << 10) + u32::from(low)
2109 - (((0xD800u32 << 10) - 0x10000u32) + 0xDC00u32);
2110 // Successfully-paired surrogate. Read from the trie again.
2111 trie_value = {
2112 // Semantically, this bit of conditional compilation makes no sense.
2113 // The purpose is to keep LLVM seeing the untyped trie case the way
2114 // it did before so as not to regress the performance of the untyped
2115 // case due to unintuitive optimizer effects. If you care about the
2116 // perf of the untyped trie case and have better ideas, please try
2117 // something better.
2118 #[cfg(not(icu4x_unstable_fast_trie_only))]
2119 {decomposition.trie.get32(upcoming32)}
2120 #[cfg(icu4x_unstable_fast_trie_only)]
2121 {decomposition.trie.get32_supplementary(upcoming32)}
2122 };
2123 if likely(starter_and_decomposes_to_self_impl(trie_value)) {
2124 continue 'fast;
2125 }
2126 break 'surrogateloop;
2127 // } else {
2128 // code_unit_iter = iter_backup;
2129 }
2130 }
2131 }
2132 // unpaired surrogate
2133 upcoming32 = 0xFFFD; // Safe value for `char::from_u32_unchecked` and matches later potential error check.
2134 // trie_value already holds a decomposition to U+FFFD.
2135 break 'surrogateloop;
2136 }
2137
2138 let upcoming = unsafe { char::from_u32_unchecked(upcoming32) };
2139 let upcoming_with_trie_value = CharacterAndTrieValue::new(upcoming, trie_value);
2140
2141
2142 let Some(consumed_so_far_slice) = pending_slice.get(..pending_slice.len() -
2143 // code_unit_iter.as_slice().len()
2144 // SAFETY: `ptr` and `end` have been derived from the same allocation
2145 // and `ptr` is never greater than `end`.
2146 unsafe { end.offset_from(ptr) as usize }
2147 - upcoming.len_utf16()) else {
2148 // If we ever come here, it's a bug, but let's avoid panic code paths in release builds.
2149 debug_assert!(false);
2150 // Throw away the results of the fast path.
2151 break 'fastwrap;
2152 };
2153 sink.write_slice(consumed_so_far_slice)?;
2154
2155 if decomposition_starts_with_non_starter(
2156 upcoming_with_trie_value.trie_val,
2157 ) {
2158 // Sync with main iterator
2159 // decomposition.delegate = code_unit_iter.as_slice().chars();
2160 // SAFETY: `ptr` and `end` have been derived from the same allocation
2161 // and `ptr` is never greater than `end`.
2162 decomposition.delegate = unsafe { core::slice::from_raw_parts(ptr, end.offset_from(ptr) as usize) }.chars();
2163 // Let this trie value to be reprocessed in case it is
2164 // one of the rare decomposing ones.
2165 decomposition.pending = Some(upcoming_with_trie_value);
2166 decomposition.gather_and_sort_combining(0);
2167 continue 'outer;
2168 }
2169 undecomposed_starter = upcoming_with_trie_value;
2170 debug_assert!(decomposition.pending.is_none());
2171 break 'fast;
2172 }
2173 // End of stream
2174 sink.write_slice(pending_slice)?;
2175 return Ok(());
2176 }
2177 // Sync the main iterator
2178 // decomposition.delegate = code_unit_iter.as_slice().chars();
2179 // SAFETY: `ptr` and `end` have been derived from the same allocation
2180 // and `ptr` is never greater than `end`.
2181 decomposition.delegate = unsafe { core::slice::from_raw_parts(ptr, end.offset_from(ptr) as usize) }.chars();
2182 break 'fastwrap;
2183 }
2184 },
2185 text,
2186 sink,
2187 decomposition,
2188 decomposition_passthrough_bound,
2189 undecomposed_starter,
2190 pending_slice,
2191 'outer,
2192 );
2193}
2194
2195/// A normalizer for performing decomposing normalization.
2196#[derive(Debug)]
2197pub struct DecomposingNormalizer {
2198 decompositions: DataPayload<NormalizerNfdDataV1>,
2199 tables: DataPayload<NormalizerNfdTablesV1>,
2200 supplementary_tables: Option<DataPayload<NormalizerNfkdTablesV1>>,
2201 decomposition_passthrough_bound: u8, // never above 0xC0
2202 composition_passthrough_bound: u16, // never above 0x0300
2203}
2204
2205impl DecomposingNormalizer {
2206 /// Constructs a borrowed version of this type for more efficient querying.
2207 pub fn as_borrowed(&self) -> DecomposingNormalizerBorrowed<'_> {
2208 DecomposingNormalizerBorrowed {
2209 decompositions: self.decompositions.get(),
2210 tables: self.tables.get(),
2211 supplementary_tables: self.supplementary_tables.as_ref().map(|s| s.get()),
2212 decomposition_passthrough_bound: self.decomposition_passthrough_bound,
2213 composition_passthrough_bound: self.composition_passthrough_bound,
2214 }
2215 }
2216
2217 /// NFD constructor using compiled data.
2218 ///
2219 /// ✨ *Enabled with the `compiled_data` Cargo feature.*
2220 ///
2221 /// [📚 Help choosing a constructor](icu_provider::constructors)
2222 #[cfg(feature = "compiled_data")]
2223 pub const fn new_nfd() -> DecomposingNormalizerBorrowed<'static> {
2224 DecomposingNormalizerBorrowed::new_nfd()
2225 }
2226
2227 icu_provider::gen_buffer_data_constructors!(
2228 () -> error: DataError,
2229 functions: [
2230 new_nfd: skip,
2231 try_new_nfd_with_buffer_provider,
2232 try_new_nfd_unstable,
2233 Self,
2234 ]
2235 );
2236
2237 #[doc = icu_provider::gen_buffer_unstable_docs!(UNSTABLE, Self::new_nfd)]
2238 pub fn try_new_nfd_unstable<D>(provider: &D) -> Result<Self, DataError>
2239 where
2240 D: DataProvider<NormalizerNfdDataV1> + DataProvider<NormalizerNfdTablesV1> + ?Sized,
2241 {
2242 let decompositions: DataPayload<NormalizerNfdDataV1> =
2243 provider.load(Default::default())?.payload;
2244 let tables: DataPayload<NormalizerNfdTablesV1> = provider.load(Default::default())?.payload;
2245
2246 if tables.get().scalars16.len() + tables.get().scalars24.len() > 0xFFF {
2247 // The data is from a future where there exists a normalization flavor whose
2248 // complex decompositions take more than 0xFFF but fewer than 0x1FFF code points
2249 // of space. If a good use case from such a decomposition flavor arises, we can
2250 // dynamically change the bit masks so that the length mask becomes 0x1FFF instead
2251 // of 0xFFF and the all-non-starters mask becomes 0 instead of 0x1000. However,
2252 // since for now the masks are hard-coded, error out.
2253 return Err(
2254 DataError::custom("future extension").with_marker(NormalizerNfdTablesV1::INFO)
2255 );
2256 }
2257
2258 let cap = decompositions.get().passthrough_cap;
2259 if cap > 0x0300 {
2260 return Err(DataError::custom("invalid").with_marker(NormalizerNfdDataV1::INFO));
2261 }
2262 // SAFETY invariant: U+FFFD must have `NON_ROUND_TRIP_MARKER` set in the trie so that
2263 // UTF-8 decoding errors (which yield U+FFFD) never pass the fast-path passthrough checks
2264 // in `normalize_utf8_to` before `str::from_utf8_unchecked`.
2265 if (decompositions.get().trie.get(char::REPLACEMENT_CHARACTER) & NON_ROUND_TRIP_MARKER) == 0
2266 {
2267 return Err(DataError::custom("invalid").with_marker(NormalizerNfdDataV1::INFO));
2268 }
2269 let decomposition_capped = cap.min(0xC0);
2270 let composition_capped = cap.min(0x0300);
2271
2272 Ok(DecomposingNormalizer {
2273 decompositions,
2274 tables,
2275 supplementary_tables: None,
2276 decomposition_passthrough_bound: decomposition_capped as u8,
2277 composition_passthrough_bound: composition_capped,
2278 })
2279 }
2280
2281 icu_provider::gen_buffer_data_constructors!(
2282 () -> error: DataError,
2283 functions: [
2284 new_nfkd: skip,
2285 try_new_nfkd_with_buffer_provider,
2286 try_new_nfkd_unstable,
2287 Self,
2288 ]
2289 );
2290
2291 /// NFKD constructor using compiled data.
2292 ///
2293 /// ✨ *Enabled with the `compiled_data` Cargo feature.*
2294 ///
2295 /// [📚 Help choosing a constructor](icu_provider::constructors)
2296 #[cfg(feature = "compiled_data")]
2297 pub const fn new_nfkd() -> DecomposingNormalizerBorrowed<'static> {
2298 DecomposingNormalizerBorrowed::new_nfkd()
2299 }
2300
2301 #[doc = icu_provider::gen_buffer_unstable_docs!(UNSTABLE, Self::new_nfkd)]
2302 pub fn try_new_nfkd_unstable<D>(provider: &D) -> Result<Self, DataError>
2303 where
2304 D: DataProvider<NormalizerNfkdDataV1>
2305 + DataProvider<NormalizerNfdTablesV1>
2306 + DataProvider<NormalizerNfkdTablesV1>
2307 + ?Sized,
2308 {
2309 let decompositions: DataPayload<NormalizerNfkdDataV1> =
2310 provider.load(Default::default())?.payload;
2311 let tables: DataPayload<NormalizerNfdTablesV1> = provider.load(Default::default())?.payload;
2312 let supplementary_tables: DataPayload<NormalizerNfkdTablesV1> =
2313 provider.load(Default::default())?.payload;
2314
2315 if tables.get().scalars16.len()
2316 + tables.get().scalars24.len()
2317 + supplementary_tables.get().scalars16.len()
2318 + supplementary_tables.get().scalars24.len()
2319 > 0xFFF
2320 {
2321 // The data is from a future where there exists a normalization flavor whose
2322 // complex decompositions take more than 0xFFF but fewer than 0x1FFF code points
2323 // of space. If a good use case from such a decomposition flavor arises, we can
2324 // dynamically change the bit masks so that the length mask becomes 0x1FFF instead
2325 // of 0xFFF and the all-non-starters mask becomes 0 instead of 0x1000. However,
2326 // since for now the masks are hard-coded, error out.
2327 return Err(
2328 DataError::custom("future extension").with_marker(NormalizerNfdTablesV1::INFO)
2329 );
2330 }
2331
2332 let cap = decompositions.get().passthrough_cap;
2333 if cap > 0x0300 {
2334 return Err(DataError::custom("invalid").with_marker(NormalizerNfkdDataV1::INFO));
2335 }
2336 // SAFETY invariant: U+FFFD must have `NON_ROUND_TRIP_MARKER` set in the trie so that
2337 // UTF-8 decoding errors (which yield U+FFFD) never pass the fast-path passthrough checks
2338 // in `normalize_utf8_to` before `str::from_utf8_unchecked`.
2339 if (decompositions.get().trie.get(char::REPLACEMENT_CHARACTER) & NON_ROUND_TRIP_MARKER) == 0
2340 {
2341 return Err(DataError::custom("invalid").with_marker(NormalizerNfkdDataV1::INFO));
2342 }
2343 let decomposition_capped = cap.min(0xC0);
2344 let composition_capped = cap.min(0x0300);
2345
2346 Ok(DecomposingNormalizer {
2347 decompositions: decompositions.cast(),
2348 tables,
2349 supplementary_tables: Some(supplementary_tables),
2350 decomposition_passthrough_bound: decomposition_capped as u8,
2351 composition_passthrough_bound: composition_capped,
2352 })
2353 }
2354
2355 /// UTS 46 decomposed constructor (testing only)
2356 ///
2357 /// This is a special building block normalization for IDNA. It is the decomposed counterpart of
2358 /// ICU4C's UTS 46 normalization with two exceptions: characters that UTS 46 disallows and
2359 /// ICU4C maps to U+FFFD and characters that UTS 46 maps to the empty string normalize as in
2360 /// NFD in this normalization. In both cases, the previous UTS 46 processing before using
2361 /// normalization is expected to deal with these characters. Making the disallowed characters
2362 /// behave like this is beneficial to data size, and this normalizer implementation cannot
2363 /// deal with a character normalizing to the empty string, which doesn't happen in NFD or
2364 /// NFKD as of Unicode 14.
2365 ///
2366 /// Warning: In this normalization, U+0345 COMBINING GREEK YPOGEGRAMMENI exhibits a behavior
2367 /// that no character in Unicode exhibits in NFD, NFKD, NFC, or NFKC: Case folding turns
2368 /// U+0345 from a reordered character into a non-reordered character before reordering happens.
2369 /// Therefore, the output of this normalization may differ for different inputs that are
2370 /// canonically equivalent with each other if they differ by how U+0345 is ordered relative
2371 /// to other reorderable characters.
2372 pub(crate) fn try_new_uts46_decomposed_unstable<D>(provider: &D) -> Result<Self, DataError>
2373 where
2374 D: DataProvider<NormalizerUts46DataV1>
2375 + DataProvider<NormalizerNfdTablesV1>
2376 + DataProvider<NormalizerNfkdTablesV1>
2377 // UTS 46 tables merged into CompatibilityDecompositionTablesV1
2378 + ?Sized,
2379 {
2380 let decompositions: DataPayload<NormalizerUts46DataV1> =
2381 provider.load(Default::default())?.payload;
2382 let tables: DataPayload<NormalizerNfdTablesV1> = provider.load(Default::default())?.payload;
2383 let supplementary_tables: DataPayload<NormalizerNfkdTablesV1> =
2384 provider.load(Default::default())?.payload;
2385
2386 if tables.get().scalars16.len()
2387 + tables.get().scalars24.len()
2388 + supplementary_tables.get().scalars16.len()
2389 + supplementary_tables.get().scalars24.len()
2390 > 0xFFF
2391 {
2392 // The data is from a future where there exists a normalization flavor whose
2393 // complex decompositions take more than 0xFFF but fewer than 0x1FFF code points
2394 // of space. If a good use case from such a decomposition flavor arises, we can
2395 // dynamically change the bit masks so that the length mask becomes 0x1FFF instead
2396 // of 0xFFF and the all-non-starters mask becomes 0 instead of 0x1000. However,
2397 // since for now the masks are hard-coded, error out.
2398 return Err(
2399 DataError::custom("future extension").with_marker(NormalizerNfdTablesV1::INFO)
2400 );
2401 }
2402
2403 let cap = decompositions.get().passthrough_cap;
2404 if cap > 0x0300 {
2405 return Err(DataError::custom("invalid").with_marker(NormalizerUts46DataV1::INFO));
2406 }
2407 // SAFETY invariant: U+FFFD must have `NON_ROUND_TRIP_MARKER` set in the trie so that
2408 // UTF-8 decoding errors (which yield U+FFFD) never pass the fast-path passthrough checks
2409 // in `normalize_utf8_to` before `str::from_utf8_unchecked`.
2410 if (decompositions.get().trie.get(char::REPLACEMENT_CHARACTER) & NON_ROUND_TRIP_MARKER) == 0
2411 {
2412 return Err(DataError::custom("invalid").with_marker(NormalizerUts46DataV1::INFO));
2413 }
2414 let decomposition_capped = cap.min(0xC0);
2415 let composition_capped = cap.min(0x0300);
2416
2417 Ok(DecomposingNormalizer {
2418 decompositions: decompositions.cast(),
2419 tables,
2420 supplementary_tables: Some(supplementary_tables),
2421 decomposition_passthrough_bound: decomposition_capped as u8,
2422 composition_passthrough_bound: composition_capped,
2423 })
2424 }
2425}
2426
2427/// Borrowed version of a normalizer for performing composing normalization.
2428#[derive(Debug)]
2429pub struct ComposingNormalizerBorrowed<'a> {
2430 decomposing_normalizer: DecomposingNormalizerBorrowed<'a>,
2431 canonical_compositions: &'a CanonicalCompositions<'a>,
2432}
2433
2434impl ComposingNormalizerBorrowed<'static> {
2435 /// Cheaply converts a [`ComposingNormalizerBorrowed<'static>`] into a [`ComposingNormalizer`].
2436 ///
2437 /// Note: Due to branching and indirection, using [`ComposingNormalizer`] might inhibit some
2438 /// compile-time optimizations that are possible with [`ComposingNormalizerBorrowed`].
2439 pub const fn static_to_owned(self) -> ComposingNormalizer {
2440 ComposingNormalizer {
2441 decomposing_normalizer: self.decomposing_normalizer.static_to_owned(),
2442 canonical_compositions: DataPayload::from_static_ref(self.canonical_compositions),
2443 }
2444 }
2445
2446 /// NFC constructor using compiled data.
2447 ///
2448 /// ✨ *Enabled with the `compiled_data` Cargo feature.*
2449 ///
2450 /// [📚 Help choosing a constructor](icu_provider::constructors)
2451 #[cfg(feature = "compiled_data")]
2452 pub const fn new_nfc() -> Self {
2453 ComposingNormalizerBorrowed {
2454 decomposing_normalizer: DecomposingNormalizerBorrowed::new_nfd(),
2455 canonical_compositions: provider::Baked::SINGLETON_NORMALIZER_NFC_V1,
2456 }
2457 }
2458
2459 /// NFKC constructor using compiled data.
2460 ///
2461 /// ✨ *Enabled with the `compiled_data` Cargo feature.*
2462 ///
2463 /// [📚 Help choosing a constructor](icu_provider::constructors)
2464 #[cfg(feature = "compiled_data")]
2465 pub const fn new_nfkc() -> Self {
2466 ComposingNormalizerBorrowed {
2467 decomposing_normalizer: DecomposingNormalizerBorrowed::new_nfkd(),
2468 canonical_compositions: provider::Baked::SINGLETON_NORMALIZER_NFC_V1,
2469 }
2470 }
2471
2472 /// This is a special building block normalization for IDNA that implements parts of the Map
2473 /// step and the following Normalize step.
2474 ///
2475 /// Warning: In this normalization, U+0345 COMBINING GREEK YPOGEGRAMMENI exhibits a behavior
2476 /// that no character in Unicode exhibits in NFD, NFKD, NFC, or NFKC: Case folding turns
2477 /// U+0345 from a reordered character into a non-reordered character before reordering happens.
2478 /// Therefore, the output of this normalization may differ for different inputs that are
2479 /// canonically equivalents with each other if they differ by how U+0345 is ordered relative
2480 /// to other reorderable characters.
2481 #[cfg(feature = "compiled_data")]
2482 pub(crate) const fn new_uts46() -> Self {
2483 ComposingNormalizerBorrowed {
2484 decomposing_normalizer: DecomposingNormalizerBorrowed::new_uts46_decomposed(),
2485 canonical_compositions: provider::Baked::SINGLETON_NORMALIZER_NFC_V1,
2486 }
2487 }
2488}
2489
2490impl<'data> ComposingNormalizerBorrowed<'data> {
2491 /// Wraps a delegate iterator into a composing iterator
2492 /// adapter by using the data already held by this normalizer.
2493 pub fn normalize_iter<I: Iterator<Item = char>>(&self, iter: I) -> Composition<'data, I> {
2494 self.normalize_iter_private(iter, IgnorableBehavior::Unsupported)
2495 }
2496
2497 fn normalize_iter_private<I: Iterator<Item = char>>(
2498 &self,
2499 iter: I,
2500 ignorable_behavior: IgnorableBehavior,
2501 ) -> Composition<'data, I> {
2502 Composition::new(
2503 Decomposition::new_with_supplements(
2504 iter,
2505 self.decomposing_normalizer.decompositions,
2506 self.decomposing_normalizer.tables,
2507 self.decomposing_normalizer.supplementary_tables,
2508 self.decomposing_normalizer.decomposition_passthrough_bound,
2509 ignorable_behavior,
2510 ),
2511 self.canonical_compositions.canonical_compositions.clone(),
2512 self.decomposing_normalizer.composition_passthrough_bound,
2513 )
2514 }
2515
2516 normalizer_methods!();
2517
2518 composing_normalize_to!(
2519 /// Normalize a string slice into a `Write` sink.
2520 ,
2521 normalize_to,
2522 core::fmt::Write,
2523 &str,
2524 {},
2525 true,
2526 as_str,
2527 {
2528 // Let's hope LICM hoists this outside `'outer`.
2529 let composition_passthrough_byte_bound = if composition_passthrough_bound == 0x300 {
2530 0xCCu8
2531 } else {
2532 // We can make this fancy if a normalization other than NFC where looking at
2533 // non-ASCII lead bytes is worthwhile is ever introduced.
2534 composition_passthrough_bound.min(0x80) as u8
2535 };
2536 // Attributes have to be on blocks, so hoisting all the way here.
2537 #[expect(clippy::unwrap_used)]
2538 'fast: loop {
2539 let mut code_unit_iter = composition.decomposition.delegate.as_str().as_bytes().iter();
2540 'fastest: loop {
2541 if let Some(&upcoming_byte) = code_unit_iter.next() {
2542 if upcoming_byte < composition_passthrough_byte_bound {
2543 // Fast-track succeeded!
2544 continue 'fastest;
2545 }
2546 let Some(remaining_slice) = pending_slice.get(pending_slice.len() - code_unit_iter.as_slice().len() - 1..) else {
2547 // If we ever come here, it's an internal bug. Let's avoid panic code paths in release builds.
2548 debug_assert!(false);
2549 // Throw away the fastest-path result in case of an internal bug.
2550 break 'fastest;
2551 };
2552 composition.decomposition.delegate = remaining_slice.chars();
2553 break 'fastest;
2554 }
2555 // End of stream
2556 sink.write_str(pending_slice)?;
2557 return Ok(());
2558 }
2559 // `unwrap()` OK, because the slice is valid UTF-8 and we know there
2560 // is an upcoming byte.
2561 let upcoming = composition.decomposition.delegate.next().unwrap();
2562 let upcoming_with_trie_value = composition.decomposition.attach_trie_value(upcoming);
2563 if upcoming_with_trie_value.potential_passthrough_and_cannot_combine_backwards() {
2564 // Can't combine backwards, hence a plain (non-backwards-combining)
2565 // starter albeit past `composition_passthrough_bound`
2566
2567 // Fast-track succeeded!
2568 continue 'fast;
2569 }
2570 // We need to fall off the fast path.
2571 composition.decomposition.pending = Some(upcoming_with_trie_value);
2572
2573 // slicing and unwrap OK, because we've just evidently read enough previously.
2574 let mut consumed_so_far = pending_slice[..pending_slice.len() - composition.decomposition.delegate.as_str().len() - upcoming.len_utf8()].chars();
2575 // `unwrap` OK, because we've previously manage to read the previous character
2576 undecomposed_starter = composition.decomposition.attach_trie_value(consumed_so_far.next_back().unwrap());
2577 let consumed_so_far_slice = consumed_so_far.as_str();
2578 sink.write_str(consumed_so_far_slice)?;
2579 break 'fast;
2580 }
2581 },
2582 text,
2583 sink,
2584 composition,
2585 composition_passthrough_bound,
2586 undecomposed_starter,
2587 pending_slice,
2588 len_utf8,
2589 );
2590
2591 composing_normalize_to!(
2592 /// Normalize a slice of potentially-invalid UTF-8 into a `Write` sink.
2593 ///
2594 /// Ill-formed byte sequences are mapped to the REPLACEMENT CHARACTER
2595 /// according to the WHATWG Encoding Standard.
2596 ///
2597 /// ✨ *Enabled with the `utf8_iter` Cargo feature.*
2598 #[cfg(feature = "utf8_iter")]
2599 ,
2600 normalize_utf8_to,
2601 core::fmt::Write,
2602 &[u8],
2603 {},
2604 false,
2605 as_slice,
2606 {
2607 'fast: loop {
2608 if let Some(upcoming) = composition.decomposition.delegate.next() {
2609 if u32::from(upcoming) < composition_passthrough_bound {
2610 // Fast-track succeeded!
2611 continue 'fast;
2612 }
2613 // TODO: Be statically aware of fast/small trie.
2614 let upcoming_with_trie_value = composition.decomposition.attach_trie_value(upcoming);
2615 if upcoming_with_trie_value.potential_passthrough_and_cannot_combine_backwards() {
2616 // Note: The trie value of the REPLACEMENT CHARACTER is
2617 // intentionally formatted to fail the
2618 // `potential_passthrough_and_cannot_combine_backwards`
2619 // test even though it really is a starter that decomposes
2620 // to self and cannot combine backwards. This
2621 // Allows moving the branch on REPLACEMENT CHARACTER
2622 // below this `continue`.
2623 continue 'fast;
2624 }
2625 // We need to fall off the fast path.
2626
2627 // TODO(#2006): Annotate as unlikely
2628 if upcoming == char::REPLACEMENT_CHARACTER {
2629 // Can't tell if this is an error or a literal U+FFFD in
2630 // the input. Assuming the former to be sure.
2631
2632 // Since the U+FFFD might signify an error, we can't
2633 // assume `upcoming.len_utf8()` for the backoff length.
2634 #[expect(clippy::indexing_slicing)]
2635 let mut consumed_so_far = pending_slice[..pending_slice.len() - composition.decomposition.delegate.as_slice().len()].chars();
2636 let back = consumed_so_far.next_back();
2637 debug_assert_eq!(back, Some(char::REPLACEMENT_CHARACTER));
2638 let consumed_so_far_slice = consumed_so_far.as_slice();
2639 sink.write_str(unsafe { str::from_utf8_unchecked(consumed_so_far_slice) })?;
2640 undecomposed_starter = CharacterAndTrieValue::new(char::REPLACEMENT_CHARACTER, 0);
2641 composition.decomposition.pending = None;
2642 break 'fast;
2643 }
2644
2645 composition.decomposition.pending = Some(upcoming_with_trie_value);
2646 // slicing and unwrap OK, because we've just evidently read enough previously.
2647 // `unwrap` OK, because we've previously manage to read the previous character
2648 #[expect(clippy::indexing_slicing)]
2649 let mut consumed_so_far = pending_slice[..pending_slice.len() - composition.decomposition.delegate.as_slice().len() - upcoming.len_utf8()].chars();
2650 #[expect(clippy::unwrap_used)]
2651 {
2652 // TODO: If the previous character was below the passthrough bound,
2653 // we really need to read from the trie. Otherwise, we could maintain
2654 // the most-recent trie value. Need to measure what's more expensive:
2655 // Remembering the trie value on each iteration or re-reading the
2656 // last one after the fast-track run.
2657 undecomposed_starter = composition.decomposition.attach_trie_value(consumed_so_far.next_back().unwrap());
2658 }
2659 let consumed_so_far_slice = consumed_so_far.as_slice();
2660 sink.write_str(unsafe { str::from_utf8_unchecked(consumed_so_far_slice)})?;
2661 break 'fast;
2662 }
2663 // End of stream
2664 sink.write_str(unsafe { str::from_utf8_unchecked(pending_slice) })?;
2665 return Ok(());
2666 }
2667 },
2668 text,
2669 sink,
2670 composition,
2671 composition_passthrough_bound,
2672 undecomposed_starter,
2673 pending_slice,
2674 len_utf8,
2675 );
2676
2677 composing_normalize_to!(
2678 /// Normalize a slice of potentially-invalid UTF-16 into a `Write16` sink.
2679 ///
2680 /// Unpaired surrogates are mapped to the REPLACEMENT CHARACTER
2681 /// before normalizing.
2682 ///
2683 /// ✨ *Enabled with the `utf16_iter` Cargo feature.*
2684 #[cfg(feature = "utf16_iter")]
2685 ,
2686 normalize_utf16_to,
2687 write16::Write16,
2688 &[u16],
2689 {
2690 sink.size_hint(text.len())?;
2691 },
2692 false,
2693 as_slice,
2694 {
2695 // This loop is only broken out of as goto forward and only as release-build recovery from
2696 // detecting an internal bug without panic. (In debug builds, internal bugs panic instead.)
2697 #[expect(clippy::never_loop)]
2698 'fastwrap: loop {
2699 // Commented out `code_unit_iter` and used `ptr` and `end` to
2700 // work around https://github.com/rust-lang/rust/issues/144684 .
2701 //
2702 // let mut code_unit_iter = composition.decomposition.delegate.as_slice().iter();
2703 let delegate_as_slice = composition.decomposition.delegate.as_slice();
2704 let mut ptr: *const u16 = delegate_as_slice.as_ptr();
2705 // SAFETY: materializing a pointer immediately past the end of an
2706 // allocation is OK.
2707 let end: *const u16 = unsafe { ptr.add(delegate_as_slice.len()) };
2708
2709 'fast: loop {
2710 // if let Some(&upcoming_code_unit) = code_unit_iter.next() {
2711 if ptr != end {
2712 // SAFETY: We just checked that `ptr` has not reached `end`.
2713 // `ptr` always advances by one, and we always have a check
2714 // per advancement.
2715 let upcoming_code_unit = unsafe { *ptr };
2716 // SAFETY: Since `ptr` hadn't reached `end`, yet, advancing
2717 // by one points to the same allocation or to immediately
2718 // after, which is OK.
2719 ptr = unsafe { ptr.add(1) };
2720
2721 let mut upcoming32 = u32::from(upcoming_code_unit); // may be surrogate
2722 // The performance of what logically is supposed to be this
2723 // branch is somewhat brittle and what LLVM ends up doing
2724 // that affects the performance of what's logically about this
2725 // decision can swing to double/halve the throughput for Basic
2726 // Latin in ways that are completely unintuitive. Basically _any_
2727 // change to _any_ code that participates in how LLVM sees the
2728 // code around here can make the perf fall over. In seems that
2729 // manually annotating this branch as likely has worse effects
2730 // on non-Basic-Latin input that the case where LLVM just happens to
2731 // do the right thing.
2732 //
2733 // What happens with this branch may depend on what sink type
2734 // this code is monomorphized over.
2735 //
2736 // What a terrible sink of developer time!
2737 if upcoming32 < composition_passthrough_bound {
2738 // No need for surrogate or U+FFFD check, because
2739 // `composition_passthrough_bound` cannot be higher than
2740 // U+0300.
2741 // Fast-track succeeded!
2742 continue 'fast;
2743 }
2744 // We might be doing a trie lookup by surrogate. Surrogates get
2745 // a decomposition to U+FFFD.
2746 let mut trie_value = composition.decomposition.trie.get16(upcoming_code_unit);
2747 if potential_passthrough_and_cannot_combine_backwards_impl(trie_value) {
2748 // Can't combine backwards, hence a plain (non-backwards-combining)
2749 // starter albeit past `composition_passthrough_bound`
2750
2751 // Fast-track succeeded!
2752 continue 'fast;
2753 }
2754
2755 // We might now be looking at a surrogate.
2756 // The loop is only broken out of as goto forward
2757 #[expect(clippy::never_loop)]
2758 'surrogateloop: loop {
2759 // The `likely` annotations _below_ exist to make the code _above_
2760 // go faster!
2761 let surrogate_base = upcoming32.wrapping_sub(0xD800);
2762 if likely(surrogate_base > (0xDFFF - 0xD800)) {
2763 // Not surrogate
2764 break 'surrogateloop;
2765 }
2766 if likely(surrogate_base <= (0xDBFF - 0xD800)) {
2767 // let iter_backup = code_unit_iter.clone();
2768 // if let Some(&low) = code_unit_iter.next() {
2769 if ptr != end {
2770 // SAFETY: We just checked that `ptr` has not reached `end`.
2771 // `ptr` always advances by one, and we always have a check
2772 // per advancement.
2773 let low = unsafe { *ptr };
2774 if likely(in_inclusive_range16(low, 0xDC00, 0xDFFF)) {
2775 // SAFETY: Since `ptr` hadn't reached `end`, yet, advancing
2776 // by one points to the same allocation or to immediately
2777 // after, which is OK.
2778 ptr = unsafe { ptr.add(1) };
2779
2780 upcoming32 = (upcoming32 << 10) + u32::from(low)
2781 - (((0xD800u32 << 10) - 0x10000u32) + 0xDC00u32);
2782 // Successfully-paired surrogate. Read from the trie again.
2783 trie_value = {
2784 // Semantically, this bit of conditional compilation makes no sense.
2785 // The purpose is to keep LLVM seeing the untyped trie case the way
2786 // it did before so as not to regress the performance of the untyped
2787 // case due to unintuitive optimizer effects. If you care about the
2788 // perf of the untyped trie case and have better ideas, please try
2789 // something better.
2790 #[cfg(not(icu4x_unstable_fast_trie_only))]
2791 {composition.decomposition.trie.get32(upcoming32)}
2792 #[cfg(icu4x_unstable_fast_trie_only)]
2793 {composition.decomposition.trie.get32_supplementary(upcoming32)}
2794 };
2795 if likely(potential_passthrough_and_cannot_combine_backwards_impl(trie_value)) {
2796 // Fast-track succeeded!
2797 continue 'fast;
2798 }
2799 break 'surrogateloop;
2800 // } else {
2801 // code_unit_iter = iter_backup;
2802 }
2803 }
2804 }
2805 // unpaired surrogate
2806 upcoming32 = 0xFFFD; // Safe value for `char::from_u32_unchecked` and matches later potential error check.
2807 // trie_value already holds a decomposition to U+FFFD.
2808 debug_assert_eq!(trie_value, NON_ROUND_TRIP_MARKER | BACKWARD_COMBINING_MARKER | 0xFFFD);
2809 break 'surrogateloop;
2810 }
2811
2812 // SAFETY: upcoming32 can no longer be a surrogate.
2813 let upcoming = unsafe { char::from_u32_unchecked(upcoming32) };
2814 let upcoming_with_trie_value = CharacterAndTrieValue::new(upcoming, trie_value);
2815 // We need to fall off the fast path.
2816 composition.decomposition.pending = Some(upcoming_with_trie_value);
2817 let Some(consumed_so_far_slice) = pending_slice.get(..pending_slice.len() -
2818 // code_unit_iter.as_slice().len()
2819 // SAFETY: `ptr` and `end` have been derived from the same allocation
2820 // and `ptr` is never greater than `end`.
2821 unsafe { end.offset_from(ptr) as usize }
2822 - upcoming.len_utf16()) else {
2823 // If we ever come here, it's a bug, but let's avoid panic code paths in release builds.
2824 debug_assert!(false);
2825 // Throw away the results of the fast path.
2826 break 'fastwrap;
2827 };
2828 let mut consumed_so_far = consumed_so_far_slice.chars();
2829 let Some(c_from_back) = consumed_so_far.next_back() else {
2830 // If we ever come here, it's a bug, but let's avoid panic code paths in release builds.
2831 debug_assert!(false);
2832 // Throw away the results of the fast path.
2833 break 'fastwrap;
2834 };
2835 // TODO: If the previous character was below the passthrough bound,
2836 // we really need to read from the trie. Otherwise, we could maintain
2837 // the most-recent trie value. Need to measure what's more expensive:
2838 // Remembering the trie value on each iteration or re-reading the
2839 // last one after the fast-track run.
2840 undecomposed_starter = composition.decomposition.attach_trie_value(c_from_back);
2841 sink.write_slice(consumed_so_far.as_slice())?;
2842 break 'fast;
2843 }
2844 // End of stream
2845 sink.write_slice(pending_slice)?;
2846 return Ok(());
2847 }
2848 // Sync the main iterator
2849 // composition.decomposition.delegate = code_unit_iter.as_slice().chars();
2850 // SAFETY: `ptr` and `end` have been derive from the same allocation
2851 // and `ptr` is never greater than `end`.
2852 composition.decomposition.delegate = unsafe { core::slice::from_raw_parts(ptr, end.offset_from(ptr) as usize) }.chars();
2853 break 'fastwrap;
2854 }
2855 },
2856 text,
2857 sink,
2858 composition,
2859 composition_passthrough_bound,
2860 undecomposed_starter,
2861 pending_slice,
2862 len_utf16,
2863 );
2864}
2865
2866/// A normalizer for performing composing normalization.
2867#[derive(Debug)]
2868pub struct ComposingNormalizer {
2869 decomposing_normalizer: DecomposingNormalizer,
2870 canonical_compositions: DataPayload<NormalizerNfcV1>,
2871}
2872
2873impl ComposingNormalizer {
2874 /// Constructs a borrowed version of this type for more efficient querying.
2875 pub fn as_borrowed(&self) -> ComposingNormalizerBorrowed<'_> {
2876 ComposingNormalizerBorrowed {
2877 decomposing_normalizer: self.decomposing_normalizer.as_borrowed(),
2878 canonical_compositions: self.canonical_compositions.get(),
2879 }
2880 }
2881
2882 /// NFC constructor using compiled data.
2883 ///
2884 /// ✨ *Enabled with the `compiled_data` Cargo feature.*
2885 ///
2886 /// [📚 Help choosing a constructor](icu_provider::constructors)
2887 #[cfg(feature = "compiled_data")]
2888 pub const fn new_nfc() -> ComposingNormalizerBorrowed<'static> {
2889 ComposingNormalizerBorrowed::new_nfc()
2890 }
2891
2892 icu_provider::gen_buffer_data_constructors!(
2893 () -> error: DataError,
2894 functions: [
2895 new_nfc: skip,
2896 try_new_nfc_with_buffer_provider,
2897 try_new_nfc_unstable,
2898 Self,
2899 ]
2900 );
2901
2902 #[doc = icu_provider::gen_buffer_unstable_docs!(UNSTABLE, Self::new_nfc)]
2903 pub fn try_new_nfc_unstable<D>(provider: &D) -> Result<Self, DataError>
2904 where
2905 D: DataProvider<NormalizerNfdDataV1>
2906 + DataProvider<NormalizerNfdTablesV1>
2907 + DataProvider<NormalizerNfcV1>
2908 + ?Sized,
2909 {
2910 let decomposing_normalizer = DecomposingNormalizer::try_new_nfd_unstable(provider)?;
2911
2912 let canonical_compositions: DataPayload<NormalizerNfcV1> =
2913 provider.load(Default::default())?.payload;
2914
2915 Ok(ComposingNormalizer {
2916 decomposing_normalizer,
2917 canonical_compositions,
2918 })
2919 }
2920
2921 /// NFKC constructor using compiled data.
2922 ///
2923 /// ✨ *Enabled with the `compiled_data` Cargo feature.*
2924 ///
2925 /// [📚 Help choosing a constructor](icu_provider::constructors)
2926 #[cfg(feature = "compiled_data")]
2927 pub const fn new_nfkc() -> ComposingNormalizerBorrowed<'static> {
2928 ComposingNormalizerBorrowed::new_nfkc()
2929 }
2930
2931 icu_provider::gen_buffer_data_constructors!(
2932 () -> error: DataError,
2933 functions: [
2934 new_nfkc: skip,
2935 try_new_nfkc_with_buffer_provider,
2936 try_new_nfkc_unstable,
2937 Self,
2938 ]
2939 );
2940
2941 #[doc = icu_provider::gen_buffer_unstable_docs!(UNSTABLE, Self::new_nfkc)]
2942 pub fn try_new_nfkc_unstable<D>(provider: &D) -> Result<Self, DataError>
2943 where
2944 D: DataProvider<NormalizerNfkdDataV1>
2945 + DataProvider<NormalizerNfdTablesV1>
2946 + DataProvider<NormalizerNfkdTablesV1>
2947 + DataProvider<NormalizerNfcV1>
2948 + ?Sized,
2949 {
2950 let decomposing_normalizer = DecomposingNormalizer::try_new_nfkd_unstable(provider)?;
2951
2952 let canonical_compositions: DataPayload<NormalizerNfcV1> =
2953 provider.load(Default::default())?.payload;
2954
2955 Ok(ComposingNormalizer {
2956 decomposing_normalizer,
2957 canonical_compositions,
2958 })
2959 }
2960
2961 #[doc = icu_provider::gen_buffer_unstable_docs!(UNSTABLE, Self::new_uts46)]
2962 pub(crate) fn try_new_uts46_unstable<D>(provider: &D) -> Result<Self, DataError>
2963 where
2964 D: DataProvider<NormalizerUts46DataV1>
2965 + DataProvider<NormalizerNfdTablesV1>
2966 + DataProvider<NormalizerNfkdTablesV1>
2967 // UTS 46 tables merged into CompatibilityDecompositionTablesV1
2968 + DataProvider<NormalizerNfcV1>
2969 + ?Sized,
2970 {
2971 let decomposing_normalizer =
2972 DecomposingNormalizer::try_new_uts46_decomposed_unstable(provider)?;
2973
2974 let canonical_compositions: DataPayload<NormalizerNfcV1> =
2975 provider.load(Default::default())?.payload;
2976
2977 Ok(ComposingNormalizer {
2978 decomposing_normalizer,
2979 canonical_compositions,
2980 })
2981 }
2982}
2983
2984#[cfg(feature = "utf16_iter")]
2985struct IsNormalizedSinkUtf16<'a> {
2986 expect: &'a [u16],
2987}
2988
2989#[cfg(feature = "utf16_iter")]
2990impl<'a> IsNormalizedSinkUtf16<'a> {
2991 pub fn new(slice: &'a [u16]) -> Self {
2992 IsNormalizedSinkUtf16 { expect: slice }
2993 }
2994 pub fn remaining_len(&self) -> usize {
2995 self.expect.len()
2996 }
2997}
2998
2999#[cfg(feature = "utf16_iter")]
3000impl write16::Write16 for IsNormalizedSinkUtf16<'_> {
3001 fn write_slice(&mut self, s: &[u16]) -> core::fmt::Result {
3002 // We know that if we get a slice, it's a pass-through,
3003 // so we can compare addresses. Indexing is OK, because
3004 // an indexing failure would be a code bug rather than
3005 // an input or data issue.
3006 #[expect(clippy::indexing_slicing)]
3007 if core::ptr::eq(s.as_ptr(), self.expect.as_ptr()) {
3008 self.expect = &self.expect[s.len()..];
3009 Ok(())
3010 } else {
3011 Err(core::fmt::Error {})
3012 }
3013 }
3014
3015 fn write_char(&mut self, c: char) -> core::fmt::Result {
3016 let mut iter = utf16_iter::ErrorReportingUtf16Chars::new(self.expect);
3017 if iter.next() == Some(Ok(c)) {
3018 self.expect = iter.as_slice();
3019 Ok(())
3020 } else {
3021 Err(core::fmt::Error {})
3022 }
3023 }
3024}
3025
3026#[cfg(feature = "utf8_iter")]
3027struct IsNormalizedSinkUtf8<'a> {
3028 expect: &'a [u8],
3029}
3030
3031#[cfg(feature = "utf8_iter")]
3032impl<'a> IsNormalizedSinkUtf8<'a> {
3033 pub fn new(slice: &'a [u8]) -> Self {
3034 IsNormalizedSinkUtf8 { expect: slice }
3035 }
3036 pub fn remaining_len(&self) -> usize {
3037 self.expect.len()
3038 }
3039}
3040
3041#[cfg(feature = "utf8_iter")]
3042impl core::fmt::Write for IsNormalizedSinkUtf8<'_> {
3043 fn write_str(&mut self, s: &str) -> core::fmt::Result {
3044 // We know that if we get a slice, it's a pass-through,
3045 // so we can compare addresses. Indexing is OK, because
3046 // an indexing failure would be a code bug rather than
3047 // an input or data issue.
3048 #[expect(clippy::indexing_slicing)]
3049 if core::ptr::eq(s.as_ptr(), self.expect.as_ptr()) {
3050 self.expect = &self.expect[s.len()..];
3051 Ok(())
3052 } else {
3053 Err(core::fmt::Error {})
3054 }
3055 }
3056
3057 fn write_char(&mut self, c: char) -> core::fmt::Result {
3058 let mut iter = utf8_iter::ErrorReportingUtf8Chars::new(self.expect);
3059 if iter.next() == Some(Ok(c)) {
3060 self.expect = iter.as_slice();
3061 Ok(())
3062 } else {
3063 Err(core::fmt::Error {})
3064 }
3065 }
3066}
3067
3068struct IsNormalizedSinkStr<'a> {
3069 expect: &'a str,
3070}
3071
3072impl<'a> IsNormalizedSinkStr<'a> {
3073 pub fn new(slice: &'a str) -> Self {
3074 IsNormalizedSinkStr { expect: slice }
3075 }
3076 pub fn remaining_len(&self) -> usize {
3077 self.expect.len()
3078 }
3079}
3080
3081impl core::fmt::Write for IsNormalizedSinkStr<'_> {
3082 fn write_str(&mut self, s: &str) -> core::fmt::Result {
3083 // We know that if we get a slice, it's a pass-through,
3084 // so we can compare addresses. Indexing is OK, because
3085 // an indexing failure would be a code bug rather than
3086 // an input or data issue.
3087 if core::ptr::eq(s.as_ptr(), self.expect.as_ptr()) {
3088 self.expect = &self.expect[s.len()..];
3089 Ok(())
3090 } else {
3091 Err(core::fmt::Error {})
3092 }
3093 }
3094
3095 fn write_char(&mut self, c: char) -> core::fmt::Result {
3096 let mut iter = self.expect.chars();
3097 if iter.next() == Some(c) {
3098 self.expect = iter.as_str();
3099 Ok(())
3100 } else {
3101 Err(core::fmt::Error {})
3102 }
3103 }
3104}