phf_macros/
lib.rs

1//! A set of macros to generate Rust source for PHF data structures at compile time.
2//! See [the `phf` crate's documentation][phf] for details.
3//!
4//! [phf]: https://docs.rs/phf
5
6#[cfg(not(feature = "ptrhash"))]
7use phf_generator::HashState;
8#[cfg(feature = "ptrhash")]
9use phf_generator::ptrhash::HashState;
10use phf_shared::PhfHash;
11use proc_macro::TokenStream;
12use quote::{ToTokens, quote};
13use std::collections::HashSet;
14use std::hash::Hasher;
15use syn::punctuated::Punctuated;
16use syn::{BinOp, Error, Expr, ExprLit, Lit, Token, UnOp, parse_macro_input};
17#[cfg(feature = "uncased")]
18use uncased_::Uncased;
19#[cfg(feature = "unicase")]
20use unicase_::{Ascii, UniCase};
21
22mod parse;
23use parse::AsMapEntry;
24
25#[derive(Hash, PartialEq, Eq, Clone)]
26enum ParsedKey {
27    Str(String),
28    Binary(Vec<u8>),
29    Char(char),
30    I8(i8),
31    I16(i16),
32    I32(i32),
33    I64(i64),
34    I128(i128),
35    Isize(isize),
36    U8(u8),
37    U16(u16),
38    U32(u32),
39    U64(u64),
40    U128(u128),
41    Usize(usize),
42    Bool(bool),
43    Tuple(Vec<ParsedKey>),
44    #[cfg(feature = "unicase")]
45    UniCase(UniCase<String>),
46    #[cfg(feature = "unicase")]
47    UniCaseAscii(Ascii<String>),
48    #[cfg(feature = "uncased")]
49    Uncased(Uncased<'static>),
50}
51
52impl PhfHash for ParsedKey {
53    fn phf_hash<H>(&self, state: &mut H)
54    where
55        H: Hasher,
56    {
57        match self {
58            ParsedKey::Str(s) => s.phf_hash(state),
59            ParsedKey::Binary(s) => s.phf_hash(state),
60            ParsedKey::Char(s) => s.phf_hash(state),
61            ParsedKey::I8(s) => s.phf_hash(state),
62            ParsedKey::I16(s) => s.phf_hash(state),
63            ParsedKey::I32(s) => s.phf_hash(state),
64            ParsedKey::I64(s) => s.phf_hash(state),
65            ParsedKey::I128(s) => s.phf_hash(state),
66            ParsedKey::Isize(s) => s.phf_hash(state),
67            ParsedKey::U8(s) => s.phf_hash(state),
68            ParsedKey::U16(s) => s.phf_hash(state),
69            ParsedKey::U32(s) => s.phf_hash(state),
70            ParsedKey::U64(s) => s.phf_hash(state),
71            ParsedKey::U128(s) => s.phf_hash(state),
72            ParsedKey::Usize(s) => s.phf_hash(state),
73            ParsedKey::Bool(s) => s.phf_hash(state),
74            ParsedKey::Tuple(elements) => {
75                for element in elements {
76                    element.phf_hash(state);
77                }
78            }
79            #[cfg(feature = "unicase")]
80            ParsedKey::UniCase(s) => s.phf_hash(state),
81            #[cfg(feature = "unicase")]
82            ParsedKey::UniCaseAscii(s) => s.phf_hash(state),
83            #[cfg(feature = "uncased")]
84            ParsedKey::Uncased(s) => s.phf_hash(state),
85        }
86    }
87}
88
89impl ParsedKey {
90    fn has_same_type_as(&self, other: &ParsedKey) -> bool {
91        match (self, other) {
92            (ParsedKey::Str(_), ParsedKey::Str(_))
93            | (ParsedKey::Binary(_), ParsedKey::Binary(_))
94            | (ParsedKey::Char(_), ParsedKey::Char(_))
95            | (ParsedKey::I8(_), ParsedKey::I8(_))
96            | (ParsedKey::I16(_), ParsedKey::I16(_))
97            | (ParsedKey::I32(_), ParsedKey::I32(_))
98            | (ParsedKey::I64(_), ParsedKey::I64(_))
99            | (ParsedKey::I128(_), ParsedKey::I128(_))
100            | (ParsedKey::Isize(_), ParsedKey::Isize(_))
101            | (ParsedKey::U8(_), ParsedKey::U8(_))
102            | (ParsedKey::U16(_), ParsedKey::U16(_))
103            | (ParsedKey::U32(_), ParsedKey::U32(_))
104            | (ParsedKey::U64(_), ParsedKey::U64(_))
105            | (ParsedKey::U128(_), ParsedKey::U128(_))
106            | (ParsedKey::Usize(_), ParsedKey::Usize(_))
107            | (ParsedKey::Bool(_), ParsedKey::Bool(_)) => true,
108            (ParsedKey::Tuple(left), ParsedKey::Tuple(right)) => {
109                left.len() == right.len()
110                    && left
111                        .iter()
112                        .zip(right)
113                        .all(|(left, right)| left.has_same_type_as(right))
114            }
115            #[cfg(feature = "unicase")]
116            (ParsedKey::UniCase(_), ParsedKey::UniCase(_)) => true,
117            #[cfg(feature = "unicase")]
118            (ParsedKey::UniCaseAscii(_), ParsedKey::UniCaseAscii(_)) => true,
119            #[cfg(feature = "uncased")]
120            (ParsedKey::Uncased(_), ParsedKey::Uncased(_)) => true,
121            _ => false,
122        }
123    }
124
125    fn from_unsuffixed_int(s: &syn::LitInt, hint: Option<&ParsedKey>) -> syn::Result<ParsedKey> {
126        match hint {
127            Some(ParsedKey::I8(_)) => Ok(ParsedKey::I8(s.base10_parse::<u8>().unwrap() as i8)),
128            Some(ParsedKey::I16(_)) => Ok(ParsedKey::I16(s.base10_parse::<u16>().unwrap() as i16)),
129            Some(ParsedKey::I32(_)) => Ok(ParsedKey::I32(s.base10_parse::<u32>().unwrap() as i32)),
130            Some(ParsedKey::I64(_)) => Ok(ParsedKey::I64(s.base10_parse::<u64>().unwrap() as i64)),
131            Some(ParsedKey::I128(_)) => {
132                Ok(ParsedKey::I128(s.base10_parse::<u128>().unwrap() as i128))
133            }
134            Some(ParsedKey::Isize(_)) => {
135                Ok(ParsedKey::Isize(s.base10_parse::<usize>().unwrap() as isize))
136            }
137            Some(ParsedKey::U8(_)) => Ok(ParsedKey::U8(s.base10_parse::<u8>().unwrap())),
138            Some(ParsedKey::U16(_)) => Ok(ParsedKey::U16(s.base10_parse::<u16>().unwrap())),
139            Some(ParsedKey::U32(_)) => Ok(ParsedKey::U32(s.base10_parse::<u32>().unwrap())),
140            Some(ParsedKey::U64(_)) => Ok(ParsedKey::U64(s.base10_parse::<u64>().unwrap())),
141            Some(ParsedKey::U128(_)) => Ok(ParsedKey::U128(s.base10_parse::<u128>().unwrap())),
142            Some(ParsedKey::Usize(_)) => Ok(ParsedKey::Usize(s.base10_parse::<usize>().unwrap())),
143            Some(_) => Err(Error::new_spanned(
144                s,
145                "integer key literal type could not be inferred from the first key",
146            )),
147            None => Err(Error::new_spanned(
148                s,
149                "integer key literals in the first key must have an explicit type suffix",
150            )),
151        }
152    }
153
154    fn from_expr(expr: &Expr, hint: Option<&ParsedKey>) -> syn::Result<ParsedKey> {
155        match expr {
156            Expr::Lit(lit) => match &lit.lit {
157                Lit::Str(s) => Ok(ParsedKey::Str(s.value())),
158                Lit::ByteStr(s) => Ok(ParsedKey::Binary(s.value())),
159                Lit::Byte(s) => Ok(ParsedKey::U8(s.value())),
160                Lit::Char(s) => Ok(ParsedKey::Char(s.value())),
161                Lit::Int(s) => match s.suffix() {
162                    // we've lost the sign at this point, so `-128i8` looks like `128i8`,
163                    // which doesn't fit in an `i8`; parse it as a `u8` and cast (to `0i8`),
164                    // which is handled below, by `Unary`
165                    "i8" => Ok(ParsedKey::I8(s.base10_parse::<u8>().unwrap() as i8)),
166                    "i16" => Ok(ParsedKey::I16(s.base10_parse::<u16>().unwrap() as i16)),
167                    "i32" => Ok(ParsedKey::I32(s.base10_parse::<u32>().unwrap() as i32)),
168                    "i64" => Ok(ParsedKey::I64(s.base10_parse::<u64>().unwrap() as i64)),
169                    "i128" => Ok(ParsedKey::I128(s.base10_parse::<u128>().unwrap() as i128)),
170                    "isize" => Ok(ParsedKey::Isize(s.base10_parse::<usize>().unwrap() as isize)),
171                    "u8" => Ok(ParsedKey::U8(s.base10_parse::<u8>().unwrap())),
172                    "u16" => Ok(ParsedKey::U16(s.base10_parse::<u16>().unwrap())),
173                    "u32" => Ok(ParsedKey::U32(s.base10_parse::<u32>().unwrap())),
174                    "u64" => Ok(ParsedKey::U64(s.base10_parse::<u64>().unwrap())),
175                    "u128" => Ok(ParsedKey::U128(s.base10_parse::<u128>().unwrap())),
176                    "usize" => Ok(ParsedKey::Usize(s.base10_parse::<usize>().unwrap())),
177                    "" => ParsedKey::from_unsuffixed_int(s, hint),
178                    _ => Err(Error::new_spanned(s, "unsupported integer literal suffix")),
179                },
180                Lit::Bool(s) => Ok(ParsedKey::Bool(s.value)),
181                _ => Err(Error::new_spanned(expr, "unsupported key expression")),
182            },
183            Expr::Array(array) => {
184                let mut buf = vec![];
185                let can_infer_u8 = matches!(hint, Some(ParsedKey::Binary(_)));
186                for expr in &array.elems {
187                    match expr {
188                        Expr::Lit(lit) => match &lit.lit {
189                            Lit::Int(s) => match s.suffix() {
190                                "u8" => buf.push(s.base10_parse::<u8>().unwrap()),
191                                "" => {
192                                    if can_infer_u8 || !buf.is_empty() {
193                                        buf.push(s.base10_parse::<u8>().unwrap());
194                                    } else {
195                                        return Err(Error::new_spanned(
196                                            s,
197                                            "integer key literals in the first key must have an explicit type suffix",
198                                        ));
199                                    }
200                                }
201                                _ => {
202                                    return Err(Error::new_spanned(
203                                        s,
204                                        "array key literals must be suffixed `u8`",
205                                    ));
206                                }
207                            },
208                            _ => {
209                                return Err(Error::new_spanned(expr, "unsupported key expression"));
210                            }
211                        },
212                        _ => return Err(Error::new_spanned(expr, "unsupported key expression")),
213                    }
214                }
215                Ok(ParsedKey::Binary(buf))
216            }
217            Expr::Unary(unary) => {
218                // Handle negation for signed integer types
219                // If we received an integer literal (always unsigned) greater than i__::max_value()
220                // then casting it to a signed integer type of the same width will negate it to
221                // the same absolute value so we don't need to negate it here
222                macro_rules! try_negate {
223                    ($val:expr) => {
224                        if $val < 0 { $val } else { -$val }
225                    };
226                }
227
228                match unary.op {
229                    UnOp::Neg(_) => match ParsedKey::from_expr(&unary.expr, hint)? {
230                        ParsedKey::I8(v) => Ok(ParsedKey::I8(try_negate!(v))),
231                        ParsedKey::I16(v) => Ok(ParsedKey::I16(try_negate!(v))),
232                        ParsedKey::I32(v) => Ok(ParsedKey::I32(try_negate!(v))),
233                        ParsedKey::I64(v) => Ok(ParsedKey::I64(try_negate!(v))),
234                        ParsedKey::I128(v) => Ok(ParsedKey::I128(try_negate!(v))),
235                        ParsedKey::Isize(v) => Ok(ParsedKey::Isize(try_negate!(v))),
236                        _ => Err(Error::new_spanned(expr, "unsupported key expression")),
237                    },
238                    UnOp::Deref(_) => {
239                        let mut expr = &*unary.expr;
240                        while let Expr::Group(group) = expr {
241                            expr = &*group.expr;
242                        }
243                        match expr {
244                            Expr::Lit(ExprLit {
245                                lit: Lit::ByteStr(s),
246                                ..
247                            }) => Ok(ParsedKey::Binary(s.value())),
248                            _ => Err(Error::new_spanned(expr, "unsupported key expression")),
249                        }
250                    }
251                    _ => Err(Error::new_spanned(expr, "unsupported key expression")),
252                }
253            }
254            Expr::Tuple(tuple) => {
255                let mut elements = Vec::new();
256                for (idx, elem) in tuple.elems.iter().enumerate() {
257                    let elem_hint = match hint {
258                        Some(ParsedKey::Tuple(hints)) => hints.get(idx),
259                        _ => None,
260                    };
261                    elements.push(ParsedKey::from_expr(elem, elem_hint)?);
262                }
263                Ok(ParsedKey::Tuple(elements))
264            }
265            Expr::Group(group) => ParsedKey::from_expr(&group.expr, hint),
266            Expr::Call(call) if call.args.len() == 1 => {
267                let last;
268                let last_ahead;
269
270                if let Expr::Path(ep) = call.func.as_ref() {
271                    let mut segments = ep.path.segments.iter();
272                    last = segments
273                        .next_back()
274                        .ok_or_else(|| Error::new_spanned(expr, "unsupported key expression"))?
275                        .ident
276                        .to_string();
277                    last_ahead = segments
278                        .next_back()
279                        .ok_or_else(|| Error::new_spanned(expr, "unsupported key expression"))?
280                        .ident
281                        .to_string();
282                } else {
283                    return Err(Error::new_spanned(expr, "unsupported key expression"));
284                }
285
286                let mut arg = call.args.first().unwrap();
287
288                while let Expr::Group(group) = arg {
289                    arg = &group.expr;
290                }
291
292                let _value = match arg {
293                    Expr::Lit(ExprLit {
294                        attrs: _,
295                        lit: Lit::Str(s),
296                    }) => s.value(),
297                    _ => {
298                        return Err(Error::new_spanned(expr, "unsupported key expression"));
299                    }
300                };
301
302                match (&*last_ahead, &*last) {
303                    #[cfg(feature = "unicase")]
304                    ("UniCase", "unicode") => Ok(ParsedKey::UniCase(UniCase::unicode(_value))),
305                    #[cfg(feature = "unicase")]
306                    ("UniCase", "ascii") => Ok(ParsedKey::UniCase(UniCase::ascii(_value))),
307                    #[cfg(feature = "unicase")]
308                    ("Ascii", "new") => Ok(ParsedKey::UniCaseAscii(Ascii::new(_value))),
309                    #[cfg(feature = "uncased")]
310                    ("UncasedStr", "new") => Ok(ParsedKey::Uncased(Uncased::new(_value))),
311                    _ => Err(Error::new_spanned(expr, "unsupported key expression")),
312                }
313            }
314            _ => Err(Error::new_spanned(expr, "unsupported key expression")),
315        }
316    }
317}
318
319fn generate_hash_state<H: PhfHash>(entries: &[H]) -> HashState {
320    #[cfg(not(feature = "ptrhash"))]
321    {
322        phf_generator::generate_hash(entries)
323    }
324
325    #[cfg(feature = "ptrhash")]
326    {
327        phf_generator::ptrhash::generate_hash(entries)
328    }
329}
330
331#[derive(Clone)]
332struct Entry {
333    parsed_key: ParsedKey,
334    key_expr: Expr,
335    value_expr: Expr,
336}
337
338impl PhfHash for Entry {
339    fn phf_hash<H>(&self, state: &mut H)
340    where
341        H: Hasher,
342    {
343        self.parsed_key.phf_hash(state)
344    }
345}
346
347struct Map {
348    entries: Vec<Entry>,
349    key_hint: Option<ParsedKey>,
350}
351
352impl Map {
353    fn from_parsed(entries: Punctuated<impl AsMapEntry, Token![,]>) -> syn::Result<Self> {
354        let mut map = Self {
355            entries: Vec::new(),
356            key_hint: None,
357        };
358        for entry in entries {
359            map.add_variants_from(&entry.key().expr, &entry.value())?;
360        }
361        map.check_duplicates()?;
362        Ok(map)
363    }
364
365    /// Handle OR patterns within the key expression
366    fn add_variants_from(&mut self, key: &Expr, value: &Expr) -> syn::Result<()> {
367        if let Expr::Binary(binary) = key {
368            if let BinOp::BitOr(_) = binary.op {
369                // Handle OR pattern: left | right
370                self.add_variants_from(&binary.left, value)?;
371                self.add_variants_from(&binary.right, value)?;
372                return Ok(());
373            }
374        }
375        // Single key
376        let parsed_key = ParsedKey::from_expr(key, self.key_hint.as_ref())?;
377        if let Some(key_hint) = &self.key_hint {
378            if !parsed_key.has_same_type_as(key_hint) {
379                return Err(Error::new_spanned(
380                    key,
381                    "key type does not match the first key",
382                ));
383            }
384        } else {
385            self.key_hint = Some(parsed_key.clone());
386        }
387        self.entries.push(Entry {
388            parsed_key,
389            key_expr: key.clone(),
390            value_expr: value.clone(),
391        });
392        Ok(())
393    }
394
395    fn check_duplicates(&self) -> syn::Result<()> {
396        let mut keys = HashSet::new();
397        for entry in &self.entries {
398            if !keys.insert(&entry.parsed_key) {
399                return Err(Error::new_spanned(&entry.key_expr, "duplicate key"));
400            }
401        }
402        Ok(())
403    }
404}
405
406fn key_has_cfg_attr(key: &parse::Key) -> bool {
407    key.attrs.iter().any(|attr| attr.path().is_ident("cfg"))
408}
409
410fn build_map(entries: &[Entry], state: HashState) -> proc_macro2::TokenStream {
411    #[cfg(not(feature = "ptrhash"))]
412    {
413        let key = state.key;
414        let disps = state.disps.iter().map(|&(d1, d2)| quote!((#d1, #d2)));
415        let entries = state.map.iter().map(|&idx| {
416            let entry = &entries[idx];
417            let key = &entry.key_expr;
418            let value = &entry.value_expr;
419            quote!((#key, #value))
420        });
421
422        quote! {
423            phf::Map {
424                key: #key,
425                disps: &[#(#disps),*],
426                entries: &[#(#entries),*],
427            }
428        }
429    }
430
431    #[cfg(feature = "ptrhash")]
432    {
433        let key = state.seed;
434        let pilots = state.pilots.iter().map(|pilot| quote!(#pilot));
435        let remap = state.remap.iter().map(|index| quote!(#index));
436        let entries = state.map.iter().map(|&idx| {
437            let entry = &entries[idx];
438            let key = &entry.key_expr;
439            let value = &entry.value_expr;
440            quote!((#key, #value))
441        });
442
443        quote! {
444            phf::Map {
445                key: #key,
446                pilots: &[#(#pilots),*],
447                remap: &[#(#remap),*],
448                entries: &[#(#entries),*],
449            }
450        }
451    }
452}
453
454fn build_ordered_map(entries: &[Entry], state: HashState) -> proc_macro2::TokenStream {
455    #[cfg(not(feature = "ptrhash"))]
456    {
457        let key = state.key;
458        let disps = state.disps.iter().map(|&(d1, d2)| quote!((#d1, #d2)));
459        let idxs = state.map.iter().map(|idx| quote!(#idx));
460        let entries = entries.iter().map(|entry| {
461            let key = &entry.key_expr;
462            let value = &entry.value_expr;
463            quote!((#key, #value))
464        });
465
466        quote! {
467            phf::OrderedMap {
468                key: #key,
469                disps: &[#(#disps),*],
470                idxs: &[#(#idxs),*],
471                entries: &[#(#entries),*],
472            }
473        }
474    }
475
476    #[cfg(feature = "ptrhash")]
477    {
478        let key = state.seed;
479        let pilots = state.pilots.iter().map(|pilot| quote!(#pilot));
480        let remap = state.remap.iter().map(|index| quote!(#index));
481        let idxs = state.map.iter().map(|idx| quote!(#idx));
482        let entries = entries.iter().map(|entry| {
483            let key = &entry.key_expr;
484            let value = &entry.value_expr;
485            quote!((#key, #value))
486        });
487
488        quote! {
489            phf::OrderedMap {
490                key: #key,
491                pilots: &[#(#pilots),*],
492                remap: &[#(#remap),*],
493                idxs: &[#(#idxs),*],
494                entries: &[#(#entries),*],
495            }
496        }
497    }
498}
499
500fn resolve_cfg<T: AsMapEntry + ToTokens>(
501    macro_name: impl ToTokens,
502    entries: Punctuated<T, Token![,]>,
503) -> TokenStream {
504    let mut cfg_args = quote! { #macro_name [] };
505
506    // Wrap conditional entries and groups of unconditional entries in { ... }.
507    // Grouping avoids unnecessarily hitting macro recursion limit. Entries are
508    // not reordered to handle ordered maps correctly (see #395).
509    let mut unconditional = Vec::new();
510    for pair in entries.pairs() {
511        let entry = pair.value();
512        if key_has_cfg_attr(entry.key()) {
513            // Pushing groups unconditionally simplifies the decl macro side.
514            quote! { { #(#unconditional)* } }.to_tokens(&mut cfg_args);
515            unconditional.clear();
516            quote! { { #pair } }.to_tokens(&mut cfg_args);
517        } else {
518            unconditional.push(pair);
519        }
520    }
521    quote! { { #(#unconditional)* } }.to_tokens(&mut cfg_args);
522
523    quote! {
524        // We generate code including paths like `phf::Map`, so accessing macros
525        // from `phf` like this should be alright.
526        phf::__resolve_cfg! {
527            #cfg_args
528        }
529    }
530    .into()
531}
532
533fn emit_code(
534    macro_name: impl ToTokens,
535    entries: Punctuated<impl AsMapEntry + ToTokens, Token![,]>,
536    builder: fn(&[Entry], HashState) -> proc_macro2::TokenStream,
537) -> TokenStream {
538    // If any entries have cfg attributes, resolve them via decl macro
539    let has_cfg_attrs = entries.iter().any(|entry| key_has_cfg_attr(entry.key()));
540    if has_cfg_attrs {
541        return resolve_cfg(macro_name, entries);
542    }
543
544    // No cfg attributes - generate code directly
545    match Map::from_parsed(entries) {
546        Ok(map) => {
547            let state = generate_hash_state(&map.entries);
548            builder(&map.entries, state).into()
549        }
550        Err(err) => err.to_compile_error().into(),
551    }
552}
553
554#[proc_macro]
555pub fn phf_map(input: TokenStream) -> TokenStream {
556    let map = parse_macro_input!(input as parse::Map);
557    emit_code(quote! { phf_map }, map.entries, build_map)
558}
559
560#[proc_macro]
561pub fn phf_set(input: TokenStream) -> TokenStream {
562    let set = parse_macro_input!(input as parse::Set);
563    emit_code(quote! { phf_set }, set.keys, |entries, state| {
564        let map = build_map(entries, state);
565        quote!(phf::Set { map: #map })
566    })
567}
568
569#[proc_macro]
570pub fn phf_ordered_map(input: TokenStream) -> TokenStream {
571    let map = parse_macro_input!(input as parse::Map);
572    emit_code(quote! { phf_ordered_map }, map.entries, build_ordered_map)
573}
574
575#[proc_macro]
576pub fn phf_ordered_set(input: TokenStream) -> TokenStream {
577    let set = parse_macro_input!(input as parse::Set);
578    emit_code(quote! { phf_ordered_set }, set.keys, |entries, state| {
579        let map = build_ordered_map(entries, state);
580        quote!(phf::OrderedSet { map: #map })
581    })
582}
phf_macros/lib.rs

phf_macros/
lib.rs
