rapidhash/v3/
rapid_const.rs

1use crate::util::hints::{assume, likely, unlikely};
2use crate::util::mix::{rapid_mix, rapid_mum};
3use crate::util::read::{read_u32, read_u64};
4use super::{DEFAULT_RAPID_SECRETS, RapidSecrets};
5
6/// Rapidhash V3 a single byte stream, matching the C++ implementation, with the default seed.
7///
8/// Fixed length inputs will greatly benefit from inlining with [rapidhash_v3_inline] instead.
9#[inline]
10pub const fn rapidhash_v3(data: &[u8]) -> u64 {
11    rapidhash_v3_inline::<true, false, false>(data, &DEFAULT_RAPID_SECRETS)
12}
13
14/// Rapidhash V3 a single byte stream, matching the C++ implementation, with a custom seed.
15///
16/// Fixed length inputs will greatly benefit from inlining with [rapidhash_v3_inline] instead.
17#[inline]
18pub const fn rapidhash_v3_seeded(data: &[u8], secrets: &RapidSecrets) -> u64 {
19    rapidhash_v3_inline::<true, false, false>(data, secrets)
20}
21
22/// Rapidhash V3 a single byte stream, matching the C++ implementation.
23///
24/// Is marked with `#[inline(always)]` to force the compiler to inline and optimize the method.
25/// Can provide large performance uplifts for fixed-length inputs at compile time.
26///
27/// Compile time arguments:
28/// - `AVALANCHE`: Perform an extra mix step to avalanche the bits for higher hash quality. Enabled
29///   by default to match the C++ implementation.
30/// - `COMPACT`: Generates fewer instructions at compile time with less manual loop unrolling, but
31///   may be slower on some platforms. Disabled by default.
32/// - `PROTECTED`: Slightly stronger hash quality and DoS resistance by performing two extra XOR
33///   instructions on every mix step. Disabled by default.
34#[inline(always)]
35pub const fn rapidhash_v3_inline<const AVALANCHE: bool, const COMPACT: bool, const PROTECTED: bool>(data: &[u8], secrets: &RapidSecrets) -> u64 {
36    rapidhash_core::<AVALANCHE, COMPACT, PROTECTED>(secrets.seed, &secrets.secrets, data)
37}
38
39/// Rapidhash V3 Micro, a very compact version of the rapidhash algorithm.
40///
41/// WARNING: This produces a different output from `rapidhash_v3`.
42///
43/// Designed for HPC and server applications, where cache misses make a noticeable performance
44/// detriment. Compiles it to ~140 instructions without stack usage, both on x86-64 and aarch64.
45/// Faster for sizes up to 512 bytes, just 15%-20% slower for inputs above 1kb.
46///
47/// Compile time arguments:
48/// - `AVALANCHE`: Perform an extra mix step to avalanche the bits for higher hash quality. Enabled
49///   by default to match the C++ implementation.
50/// - `PROTECTED`: Slightly stronger hash quality and DoS resistance by performing two extra XOR
51///   instructions on every mix step. Disabled by default.
52#[inline(always)]
53pub const fn rapidhash_v3_micro_inline<const AVALANCHE: bool, const PROTECTED: bool>(data: &[u8], seed: &RapidSecrets) -> u64 {
54    rapidhash_micro_core::<AVALANCHE, PROTECTED>(seed.seed, &seed.secrets, data)
55}
56
57/// Rapidhash V3 Nano, a very compact version of the rapidhash algorithm.
58///
59/// WARNING: This produces a different output from `rapidhash_v3`.
60///
61/// Designed for Mobile and embedded applications, where keeping a small code size is a top priority.
62/// This should compile it to less than 100 instructions with minimal stack usage, both on x86-64
63/// and aarch64. The fastest for sizes up to 48 bytes, but may be considerably slower for larger
64/// inputs.
65///
66/// Compile time arguments:
67/// - `AVALANCHE`: Perform an extra mix step to avalanche the bits for higher hash quality. Enabled
68///   by default to match the C++ implementation.
69/// - `PROTECTED`: Slightly stronger hash quality and DoS resistance by performing two extra XOR
70///   instructions on every mix step. Disabled by default.
71#[inline(always)]
72pub const fn rapidhash_v3_nano_inline<const AVALANCHE: bool, const PROTECTED: bool>(data: &[u8], seed: &RapidSecrets) -> u64 {
73    rapidhash_nano_core::<AVALANCHE, PROTECTED>(seed.seed, &seed.secrets, data)
74}
75
76#[inline(always)]
77pub(super) const fn rapidhash_core<const AVALANCHE: bool, const COMPACT: bool, const PROTECTED: bool>(mut seed: u64, secrets: &[u64; 7], data: &[u8]) -> u64 {
78    let mut a;
79    let mut b;
80
81    let remainder;
82    if likely(data.len() <= 16) {
83        a = 0;
84        b = 0;
85
86        if data.len() >= 4 {
87            seed ^= data.len() as u64;
88            if data.len() >= 8 {
89                let plast = data.len() - 8;
90                a ^= read_u64(data, 0);
91                b ^= read_u64(data, plast);
92            } else {
93                let plast = data.len() - 4;
94                a ^= read_u32(data, 0) as u64;
95                b ^= read_u32(data, plast) as u64;
96            }
97        } else if !data.is_empty() {
98            a ^= ((data[0] as u64) << 45) | data[data.len() - 1] as u64;
99            b ^= data[data.len() >> 1] as u64;
100        }
101        remainder = data.len() as u64;
102    } else {
103        // SAFETY: we have just verified that data.len() > 16
104        unsafe {
105            return rapidhash_core_cold::<AVALANCHE, COMPACT, PROTECTED>(seed, secrets, data);
106        }
107    }
108
109    a ^= secrets[1];
110    b ^= seed;
111
112    (a, b) = rapid_mum::<PROTECTED>(a, b);
113
114    if AVALANCHE {
115        rapidhash_finish::<PROTECTED>(a, b, remainder, secrets)
116    } else {
117        a ^ b
118    }
119}
120
121// This is sadly a fat function with a lot of calling overhead because it clobbers registers.
122// Great for reaching max performance on 1kB+ inputs, but not great for 25 byte
123// inputs... We therefore mark this as #[inline] to let the compiler decide whether to inline it or
124// not, if it knows the input size. If the input size is known to be <112, there's a lot to gain
125// through inlining and optimising away the 7 data-independent execution paths. The RapidHasher
126// deviates from the V3 implementation here because of this!
127#[inline]
128const unsafe fn rapidhash_core_cold<const AVALANCHE: bool, const COMPACT: bool, const PROTECTED: bool>(mut seed: u64, secrets: &[u64; 7], data: &[u8]) -> u64 {
129    // SAFETY: we promise to never call this with <=16 length data to omit some bounds checks.
130    // This is really intended for codegen-units >1 and/or no LTO.
131    assume(data.len() > 16);
132
133    let mut a = 0;
134    let mut b = 0;
135
136    let mut slice = data;
137
138    if unlikely(slice.len() > 112) {
139        // most CPUs appear to benefit from this unrolled loop
140        let mut see1 = seed;
141        let mut see2 = seed;
142        let mut see3 = seed;
143        let mut see4 = seed;
144        let mut see5 = seed;
145        let mut see6 = seed;
146
147        if !COMPACT {
148            while slice.len() > 224 {
149                seed = rapid_mix::<PROTECTED>(read_u64(slice, 0) ^ secrets[0], read_u64(slice, 8) ^ seed);
150                see1 = rapid_mix::<PROTECTED>(read_u64(slice, 16) ^ secrets[1], read_u64(slice, 24) ^ see1);
151                see2 = rapid_mix::<PROTECTED>(read_u64(slice, 32) ^ secrets[2], read_u64(slice, 40) ^ see2);
152                see3 = rapid_mix::<PROTECTED>(read_u64(slice, 48) ^ secrets[3], read_u64(slice, 56) ^ see3);
153                see4 = rapid_mix::<PROTECTED>(read_u64(slice, 64) ^ secrets[4], read_u64(slice, 72) ^ see4);
154                see5 = rapid_mix::<PROTECTED>(read_u64(slice, 80) ^ secrets[5], read_u64(slice, 88) ^ see5);
155                see6 = rapid_mix::<PROTECTED>(read_u64(slice, 96) ^ secrets[6], read_u64(slice, 104) ^ see6);
156
157                seed = rapid_mix::<PROTECTED>(read_u64(slice, 112) ^ secrets[0], read_u64(slice, 120) ^ seed);
158                see1 = rapid_mix::<PROTECTED>(read_u64(slice, 128) ^ secrets[1], read_u64(slice, 136) ^ see1);
159                see2 = rapid_mix::<PROTECTED>(read_u64(slice, 144) ^ secrets[2], read_u64(slice, 152) ^ see2);
160                see3 = rapid_mix::<PROTECTED>(read_u64(slice, 160) ^ secrets[3], read_u64(slice, 168) ^ see3);
161                see4 = rapid_mix::<PROTECTED>(read_u64(slice, 176) ^ secrets[4], read_u64(slice, 184) ^ see4);
162                see5 = rapid_mix::<PROTECTED>(read_u64(slice, 192) ^ secrets[5], read_u64(slice, 200) ^ see5);
163                see6 = rapid_mix::<PROTECTED>(read_u64(slice, 208) ^ secrets[6], read_u64(slice, 216) ^ see6);
164
165                let (_, split) = slice.split_at(224);
166                slice = split;
167            }
168
169            if slice.len() > 112 {
170                seed = rapid_mix::<PROTECTED>(read_u64(slice, 0) ^ secrets[0], read_u64(slice, 8) ^ seed);
171                see1 = rapid_mix::<PROTECTED>(read_u64(slice, 16) ^ secrets[1], read_u64(slice, 24) ^ see1);
172                see2 = rapid_mix::<PROTECTED>(read_u64(slice, 32) ^ secrets[2], read_u64(slice, 40) ^ see2);
173                see3 = rapid_mix::<PROTECTED>(read_u64(slice, 48) ^ secrets[3], read_u64(slice, 56) ^ see3);
174                see4 = rapid_mix::<PROTECTED>(read_u64(slice, 64) ^ secrets[4], read_u64(slice, 72) ^ see4);
175                see5 = rapid_mix::<PROTECTED>(read_u64(slice, 80) ^ secrets[5], read_u64(slice, 88) ^ see5);
176                see6 = rapid_mix::<PROTECTED>(read_u64(slice, 96) ^ secrets[6], read_u64(slice, 104) ^ see6);
177                let (_, split) = slice.split_at(112);
178                slice = split;
179            }
180        } else {
181            while slice.len() > 112 {
182                seed = rapid_mix::<PROTECTED>(read_u64(slice, 0) ^ secrets[0], read_u64(slice, 8) ^ seed);
183                see1 = rapid_mix::<PROTECTED>(read_u64(slice, 16) ^ secrets[1], read_u64(slice, 24) ^ see1);
184                see2 = rapid_mix::<PROTECTED>(read_u64(slice, 32) ^ secrets[2], read_u64(slice, 40) ^ see2);
185                see3 = rapid_mix::<PROTECTED>(read_u64(slice, 48) ^ secrets[3], read_u64(slice, 56) ^ see3);
186                see4 = rapid_mix::<PROTECTED>(read_u64(slice, 64) ^ secrets[4], read_u64(slice, 72) ^ see4);
187                see5 = rapid_mix::<PROTECTED>(read_u64(slice, 80) ^ secrets[5], read_u64(slice, 88) ^ see5);
188                see6 = rapid_mix::<PROTECTED>(read_u64(slice, 96) ^ secrets[6], read_u64(slice, 104) ^ see6);
189                let (_, split) = slice.split_at(112);
190                slice = split;
191            }
192        }
193
194        seed ^= see1;
195        see2 ^= see3;
196        see4 ^= see5;
197        seed ^= see6;
198        see2 ^= see4;
199        seed ^= see2;
200    }
201
202    if slice.len() > 16 {
203        seed = rapid_mix::<PROTECTED>(read_u64(slice, 0) ^ secrets[2], read_u64(slice, 8) ^ seed);
204        if slice.len() > 32 {
205            seed = rapid_mix::<PROTECTED>(read_u64(slice, 16) ^ secrets[2], read_u64(slice, 24) ^ seed);
206            if slice.len() > 48 {
207                seed = rapid_mix::<PROTECTED>(read_u64(slice, 32) ^ secrets[1], read_u64(slice, 40) ^ seed);
208                if slice.len() > 64 {
209                    seed = rapid_mix::<PROTECTED>(read_u64(slice, 48) ^ secrets[1], read_u64(slice, 56) ^ seed);
210                    if slice.len() > 80 {
211                        seed = rapid_mix::<PROTECTED>(read_u64(slice, 64) ^ secrets[2], read_u64(slice, 72) ^ seed);
212                        if slice.len() > 96 {
213                            seed = rapid_mix::<PROTECTED>(read_u64(slice, 80) ^ secrets[1], read_u64(slice, 88) ^ seed);
214                        }
215                    }
216                }
217            }
218        }
219    }
220
221    a ^= read_u64(data, data.len() - 16) ^ slice.len() as u64;
222    b ^= read_u64(data, data.len() - 8);
223
224    a ^= secrets[1];
225    b ^= seed;
226
227    (a, b) = rapid_mum::<PROTECTED>(a, b);
228
229    if AVALANCHE {
230        rapidhash_finish::<PROTECTED>(a, b, slice.len() as u64, secrets)
231    } else {
232        a ^ b
233    }
234}
235
236const fn rapidhash_micro_core<const AVALANCHE: bool, const PROTECTED: bool>(mut seed: u64, secrets: &[u64; 7], data: &[u8]) -> u64 {
237    let mut a = 0;
238    let mut b = 0;
239
240    let remainder;
241    if likely(data.len() <= 16) {
242        if data.len() >= 4 {
243            seed ^= data.len() as u64;
244            if data.len() >= 8 {
245                let plast = data.len() - 8;
246                a ^= read_u64(data, 0);
247                b ^= read_u64(data, plast);
248            } else {
249                let plast = data.len() - 4;
250                a ^= read_u32(data, 0) as u64;
251                b ^= read_u32(data, plast) as u64;
252            }
253        } else if !data.is_empty() {
254            a ^= ((data[0] as u64) << 45) | data[data.len() - 1] as u64;
255            b ^= data[data.len() >> 1] as u64;
256        }
257        remainder = data.len() as u64;
258    } else {
259        let mut slice = data;
260        if unlikely(slice.len() > 80) {
261            let mut see1 = seed;
262            let mut see2 = seed;
263            let mut see3 = seed;
264            let mut see4 = seed;
265
266            while slice.len() > 80 {
267                seed = rapid_mix::<PROTECTED>(read_u64(slice, 0) ^ secrets[0], read_u64(slice, 8) ^ seed);
268                see1 = rapid_mix::<PROTECTED>(read_u64(slice, 16) ^ secrets[1], read_u64(slice, 24) ^ see1);
269                see2 = rapid_mix::<PROTECTED>(read_u64(slice, 32) ^ secrets[2], read_u64(slice, 40) ^ see2);
270                see3 = rapid_mix::<PROTECTED>(read_u64(slice, 48) ^ secrets[3], read_u64(slice, 56) ^ see3);
271                see4 = rapid_mix::<PROTECTED>(read_u64(slice, 64) ^ secrets[4], read_u64(slice, 72) ^ see4);
272                let (_, split) = slice.split_at(80);
273                slice = split;
274            }
275
276            seed ^= see1;
277            see2 ^= see3;
278            seed ^= see4;
279            seed ^= see2;
280        }
281
282        if slice.len() > 16 {
283            seed = rapid_mix::<PROTECTED>(read_u64(slice, 0) ^ secrets[2], read_u64(slice, 8) ^ seed);
284            if slice.len() > 32 {
285                seed = rapid_mix::<PROTECTED>(read_u64(slice, 16) ^ secrets[2], read_u64(slice, 24) ^ seed);
286                if slice.len() > 48 {
287                    seed = rapid_mix::<PROTECTED>(read_u64(slice, 32) ^ secrets[1], read_u64(slice, 40) ^ seed);
288                    if slice.len() > 64 {
289                        seed = rapid_mix::<PROTECTED>(read_u64(slice, 48) ^ secrets[1], read_u64(slice, 56) ^ seed);
290                    }
291                }
292            }
293        }
294
295        remainder = slice.len() as u64;
296        a ^= read_u64(data, data.len() - 16) ^ remainder;
297        b ^= read_u64(data, data.len() - 8);
298    }
299
300    a ^= secrets[1];
301    b ^= seed;
302
303    (a, b) = rapid_mum::<PROTECTED>(a, b);
304
305    if AVALANCHE {
306        rapidhash_finish::<PROTECTED>(a, b, remainder, secrets)
307    } else {
308        a ^ b
309    }
310}
311
312const fn rapidhash_nano_core<const AVALANCHE: bool, const PROTECTED: bool>(mut seed: u64, secrets: &[u64; 7], data: &[u8]) -> u64 {
313    let mut a = 0;
314    let mut b = 0;
315
316    let remainder;
317    if likely(data.len() <= 16) {
318        if data.len() >= 4 {
319            seed ^= data.len() as u64;
320            if data.len() >= 8 {
321                let plast = data.len() - 8;
322                a ^= read_u64(data, 0);
323                b ^= read_u64(data, plast);
324            } else {
325                let plast = data.len() - 4;
326                a ^= read_u32(data, 0) as u64;
327                b ^= read_u32(data, plast) as u64;
328            }
329        } else if !data.is_empty() {
330            a ^= ((data[0] as u64) << 45) | data[data.len() - 1] as u64;
331            b ^= data[data.len() >> 1] as u64;
332        }
333        remainder = data.len() as u64;
334    } else {
335        let mut slice = data;
336        if unlikely(slice.len() > 48) {
337            let mut see1 = seed;
338            let mut see2 = seed;
339
340            while slice.len() > 48 {
341                seed = rapid_mix::<PROTECTED>(read_u64(slice, 0) ^ secrets[0], read_u64(slice, 8) ^ seed);
342                see1 = rapid_mix::<PROTECTED>(read_u64(slice, 16) ^ secrets[1], read_u64(slice, 24) ^ see1);
343                see2 = rapid_mix::<PROTECTED>(read_u64(slice, 32) ^ secrets[2], read_u64(slice, 40) ^ see2);
344                let (_, split) = slice.split_at(48);
345                slice = split;
346            }
347
348            seed ^= see1;
349            seed ^= see2;
350        }
351
352        if slice.len() > 16 {
353            seed = rapid_mix::<PROTECTED>(read_u64(slice, 0) ^ secrets[2], read_u64(slice, 8) ^ seed);
354            if slice.len() > 32 {
355                seed = rapid_mix::<PROTECTED>(read_u64(slice, 16) ^ secrets[2], read_u64(slice, 24) ^ seed);
356            }
357        }
358
359        remainder = slice.len() as u64;
360        a ^= read_u64(data, data.len() - 16) ^ remainder;
361        b ^= read_u64(data, data.len() - 8);
362    }
363
364    a ^= secrets[1];
365    b ^= seed;
366
367    (a, b) = rapid_mum::<PROTECTED>(a, b);
368    if AVALANCHE {
369        rapidhash_finish::<PROTECTED>(a, b, remainder, secrets)
370    } else {
371        a ^ b
372    }
373}
374
375#[inline(always)]
376pub(super) const fn rapidhash_finish<const PROTECTED: bool>(a: u64, b: u64, remainder: u64, secrets: &[u64; 7]) -> u64 {
377    rapid_mix::<PROTECTED>(a ^ 0xaaaaaaaaaaaaaaaa, b ^ secrets[1] ^ remainder)
378}
rapidhash/v3/rapid_const.rs

rapidhash/v3/
rapid_const.rs
