Skip to main content

tor_geoip/
lib.rs

1//! A crate for performing GeoIP lookups using the Tor GeoIP database.
2
3// @@ begin lint list maintained by maint/add_warning @@
4#![allow(renamed_and_removed_lints)] // @@REMOVE_WHEN(ci_arti_stable)
5#![allow(unknown_lints)] // @@REMOVE_WHEN(ci_arti_nightly)
6#![warn(missing_docs)]
7#![warn(noop_method_call)]
8#![warn(unreachable_pub)]
9#![warn(clippy::all)]
10#![deny(clippy::await_holding_lock)]
11#![deny(clippy::cargo_common_metadata)]
12#![deny(clippy::cast_lossless)]
13#![deny(clippy::checked_conversions)]
14#![allow(clippy::cognitive_complexity)] // See arti#2556
15#![deny(clippy::debug_assert_with_mut_call)]
16#![deny(clippy::exhaustive_enums)]
17#![deny(clippy::exhaustive_structs)]
18#![deny(clippy::expl_impl_clone_on_copy)]
19#![deny(clippy::fallible_impl_from)]
20#![deny(clippy::implicit_clone)]
21#![deny(clippy::large_stack_arrays)]
22#![warn(clippy::manual_ok_or)]
23#![deny(clippy::missing_docs_in_private_items)]
24#![warn(clippy::needless_borrow)]
25#![warn(clippy::needless_pass_by_value)]
26#![warn(clippy::option_option)]
27#![warn(clippy::or_fun_call)]
28#![deny(clippy::print_stderr)]
29#![deny(clippy::print_stdout)]
30#![warn(clippy::rc_buffer)]
31#![deny(clippy::ref_option_ref)]
32#![warn(clippy::semicolon_if_nothing_returned)]
33#![warn(clippy::trait_duplication_in_bounds)]
34#![deny(clippy::unchecked_time_subtraction)]
35#![deny(clippy::unnecessary_wraps)]
36#![warn(clippy::unseparated_literal_suffix)]
37#![deny(clippy::unwrap_used)]
38#![deny(clippy::mod_module_files)]
39#![allow(clippy::let_unit_value)] // This can reasonably be done for explicitness
40#![allow(clippy::uninlined_format_args)]
41#![allow(clippy::significant_drop_in_scrutinee)] // arti/-/merge_requests/588/#note_2812945
42#![allow(clippy::result_large_err)] // temporary workaround for arti#587
43#![allow(clippy::needless_raw_string_hashes)] // complained-about code is fine, often best
44#![allow(clippy::needless_lifetimes)] // See arti#1765
45#![allow(mismatched_lifetime_syntaxes)] // temporary workaround for arti#2060
46#![allow(clippy::collapsible_if)] // See arti#2342
47#![deny(clippy::unused_async)]
48#![deny(clippy::string_slice)] // See arti#2571
49#![allow(recursion_depth_exceeding_limit)] // arti#2715, rust/issues/159228
50//! <!-- @@ end lint list maintained by maint/add_warning @@ -->
51
52// TODO #1645 (either remove this, or decide to have it everywhere)
53#![cfg_attr(not(all(feature = "full")), allow(unused))]
54// TODO: add to the list maintained by maint/add_warning once it's available on all crates
55#![warn(unnameable_types)]
56
57use crate::dense_range_map::DenseRangeMap;
58pub use crate::err::Error;
59use std::fmt::{Debug, Display, Formatter};
60use std::net::{IpAddr, Ipv6Addr};
61use std::num::{NonZeroU16, NonZeroU32};
62use std::ops::RangeInclusive;
63use std::str::FromStr;
64use std::sync::{Arc, OnceLock};
65
66mod dense_range_map;
67mod err;
68
69#[cfg(feature = "rpc")]
70pub mod rpc;
71
72/// A parsed copy of the embedded database.
73#[cfg(feature = "embedded-db")]
74static EMBEDDED_DB_PARSED: OnceLock<Arc<GeoipDb>> = OnceLock::new();
75
76/// A two-letter country code.
77///
78/// Specifically, this type represents a purported "ISO 3166-1 alpha-2" country
79/// code, such as "IT" for Italy or "UY" for Uruguay.
80///
81/// It does not include the sentinel value `??` that we use to represent
82/// "country unknown"; if you need that, use [`OptionCc`]. Other than that, we
83/// do not check whether the country code represents a real country: we only
84/// ensure that it is a pair of printing ASCII characters.
85///
86/// Note that the geoip databases included with Arti will only include real
87/// countries; we do not include the pseudo-countries `A1` through `An` for
88/// "anonymous proxies", since doing so would mean putting nearly all Tor relays
89/// into one of those countries.
90#[derive(Copy, Clone, Eq, PartialEq)]
91#[cfg_attr(
92    feature = "serde",
93    derive(serde_with::DeserializeFromStr, serde_with::SerializeDisplay)
94)]
95#[repr(transparent)]
96pub struct CountryCode {
97    /// The underlying value (two printable ASCII characters, stored uppercase).
98    ///
99    /// The special value `??` is excluded, since it is not a country; use
100    /// `OptionCc` instead if you need to represent that.
101    ///
102    /// We store these as `NonZeroU16` so that an `Option<CountryCode>` only has to
103    /// take 2 bytes. This helps with alignment and storage.
104    ///
105    /// (We use a `NonZeroU16` rather than `[NonZeroU8; 2]` to ensure that every
106    /// bit representation is a valid `Option<CountryCode>`.)
107    inner: NonZeroU16,
108}
109
110impl CountryCode {
111    /// Make a new `CountryCode`.
112    fn new(cc_orig: &str) -> Result<Self, Error> {
113        /// Try to convert an array of 2 bytes into a NonZeroU16.
114        #[inline]
115        fn try_cvt_to_nz(inp: [u8; 2]) -> Result<NonZeroU16, Error> {
116            if inp[0] == 0 || inp[1] == 0 {
117                return Err(Error::BadCountryCode("Country code contained NULs".into()));
118            }
119            Ok(u16::from_ne_bytes(inp)
120                .try_into()
121                .expect("zero arrived surprisingly"))
122        }
123
124        let cc = cc_orig.to_ascii_uppercase();
125
126        let cc: [u8; 2] = cc
127            .as_bytes()
128            .try_into()
129            .map_err(|_| Error::BadCountryCode(cc))?;
130
131        if !cc.iter().all(|b| b.is_ascii() && !b.is_ascii_control()) {
132            return Err(Error::BadCountryCode(cc_orig.to_owned()));
133        }
134
135        if &cc == b"??" {
136            return Err(Error::NowhereNotSupported);
137        }
138
139        Ok(Self {
140            inner: try_cvt_to_nz(cc).map_err(|_| Error::BadCountryCode(cc_orig.to_owned()))?,
141        })
142    }
143
144    /// Get the actual country code.
145    ///
146    /// This just calls `.as_ref()`.
147    pub fn get(&self) -> &str {
148        self.as_ref()
149    }
150}
151
152impl Display for CountryCode {
153    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
154        write!(f, "{}", self.as_ref())
155    }
156}
157
158impl Debug for CountryCode {
159    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
160        write!(f, "CountryCode(\"{}\")", self.as_ref())
161    }
162}
163
164impl AsRef<str> for CountryCode {
165    fn as_ref(&self) -> &str {
166        /// Convert a reference to a NonZeroU16 to a reference to
167        /// an array of 2 bytes.
168        #[inline]
169        fn cvt_ref(inp: &NonZeroU16) -> &[u8; 2] {
170            // SAFETY: Every NonZeroU16 has a layout, alignment, and bit validity that is
171            // also a valid [u8; 2].  The layout of arrays is also guaranteed.
172            //
173            // (We don't use try_into here because we need to return a str that
174            // points to a reference to self.)
175            let slice: &[NonZeroU16] = std::slice::from_ref(inp);
176            let (_, slice, _) = unsafe { slice.align_to::<u8>() };
177            slice
178                .try_into()
179                .expect("the resulting slice should have the correct length!")
180        }
181
182        // This shouldn't ever panic, since we shouldn't feed non-utf8 country
183        // codes in.
184        //
185        // In theory we could use from_utf8_unchecked, but that's probably not
186        // needed.
187        std::str::from_utf8(cvt_ref(&self.inner)).expect("invalid country code in CountryCode")
188    }
189}
190
191impl FromStr for CountryCode {
192    type Err = Error;
193
194    fn from_str(s: &str) -> Result<Self, Self::Err> {
195        CountryCode::new(s)
196    }
197}
198
199/// Wrapper for an `Option<`[`CountryCode`]`>` that encodes `None` as `??`.
200///
201/// Used so that we can implement foreign traits.
202#[derive(
203    Copy, Clone, Debug, Eq, PartialEq, derive_more::Into, derive_more::From, derive_more::AsRef,
204)]
205#[allow(clippy::exhaustive_structs)]
206pub struct OptionCc(pub Option<CountryCode>);
207
208impl FromStr for OptionCc {
209    type Err = Error;
210
211    fn from_str(s: &str) -> Result<Self, Self::Err> {
212        match CountryCode::new(s) {
213            Err(Error::NowhereNotSupported) => Ok(None.into()),
214            Err(e) => Err(e),
215            Ok(cc) => Ok(Some(cc).into()),
216        }
217    }
218}
219
220impl Display for OptionCc {
221    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
222        match self.0 {
223            Some(cc) => write!(f, "{}", cc),
224            None => write!(f, "??"),
225        }
226    }
227}
228
229/// The type of an ASN.
230type Asn = NonZeroU32;
231
232/// A database of IP addresses to country codes.
233#[derive(Clone, Eq, PartialEq, Debug)]
234#[cfg_attr(feature = "rpc", derive(derive_deftly::Deftly))]
235#[cfg_attr(feature = "rpc", derive_deftly(tor_rpcbase::Object))]
236pub struct GeoipDb {
237    /// The IPv4 subset of the database, with v4 addresses stored as 32-bit integers.
238    map_v4: DenseRangeMap<u32, CountryCode, Asn>,
239    /// The IPv6 subset of the database, with v6 addresses stored as 128-bit integers.
240    map_v6: DenseRangeMap<u128, CountryCode, Asn>,
241}
242
243impl GeoipDb {
244    /// Make a new `GeoipDb` using a compiled-in copy of the GeoIP database.
245    ///
246    /// The returned instance of the database is shared with `Arc` across all invocations of this
247    /// function in the same program.
248    #[cfg(feature = "embedded-db")]
249    pub fn new_embedded() -> Arc<Self> {
250        Arc::clone(EMBEDDED_DB_PARSED.get_or_init(|| {
251            use tor_geoip_db as db;
252            fn cvt_ccs(ccs: &'static [Option<NonZeroU16>]) -> &'static [Option<CountryCode>] {
253                // SAFETY: CountryCode is a repr(transparent) for NonZeroU16.
254                let (pre, data, post) = unsafe { ccs.align_to::<Option<CountryCode>>() };
255                assert!(pre.is_empty());
256                assert!(post.is_empty());
257                data
258            }
259
260            let map_v4 = DenseRangeMap::from_static_parts(db::ipv4s(), cvt_ccs(db::ipv4c()), None);
261            let map_v6 = DenseRangeMap::from_static_parts(db::ipv6s(), cvt_ccs(db::ipv6c()), None);
262
263            Arc::new(
264                // It's reasonable to assume the one we embedded is fine --
265                // we'll test it in CI, etc.
266                GeoipDb { map_v4, map_v6 },
267            )
268        }))
269    }
270
271    /// Make a new `GeoipDb` using provided copies of the v4 and v6 database, in Tor legacy format.
272    pub fn new_from_legacy_format(
273        db_v4: &str,
274        db_v6: &str,
275        include_asn: bool,
276    ) -> Result<Self, Error> {
277        let discard_asn = !include_asn;
278        let map_v4 = DenseRangeMap::try_from_sorted_inclusive_ranges(
279            db_v4
280                .lines()
281                .filter_map(|line| parse_line::<u32>(line).transpose()),
282            discard_asn,
283        )?;
284
285        let map_v6 = DenseRangeMap::try_from_sorted_inclusive_ranges(
286            db_v6
287                .lines()
288                .filter_map(|line| parse_line::<Ipv6Addr>(line).transpose()),
289            discard_asn,
290        )?;
291
292        Ok(Self { map_v4, map_v6 })
293    }
294
295    /// Return the database in a raw format suitable for embedding.
296    ///
297    /// This method and the format it returns are unstable.
298    /// This method should only be used for maintaining the database.
299    #[cfg(feature = "export")]
300    #[allow(clippy::type_complexity)]
301    pub fn export_raw(&self) -> RawGeoipDbExport {
302        let (ipv4_starts, ipv4_ccs, ipv4_asns) = self.map_v4.export();
303        let (ipv6_starts, ipv6_ccs, ipv6_asns) = self.map_v6.export();
304
305        RawGeoipDbExport {
306            ipv4_starts,
307            ipv4_ccs,
308            ipv4_asns,
309            ipv6_starts,
310            ipv6_ccs,
311            ipv6_asns,
312        }
313    }
314
315    /// Get a 2-letter country code for the given IP address, if this data is available.
316    pub fn lookup_country_code(&self, ip: IpAddr) -> Option<&CountryCode> {
317        match ip.to_canonical() {
318            IpAddr::V4(v4) => self.map_v4.get1(&v4.into()),
319            IpAddr::V6(v6) => self.map_v6.get1(&v6.into()),
320        }
321    }
322
323    /// Determine a 2-letter country code for a host with multiple IP addresses.
324    ///
325    /// This looks up all of the IP addresses with `lookup_country_code`. If the lookups
326    /// return different countries, `None` is returned. IP addresses that fail to resolve
327    /// into a country are ignored if some of the other addresses do resolve successfully.
328    pub fn lookup_country_code_multi<I>(&self, ips: I) -> Option<&CountryCode>
329    where
330        I: IntoIterator<Item = IpAddr>,
331    {
332        let mut ret = None;
333
334        for ip in ips {
335            if let Some(cc) = self.lookup_country_code(ip) {
336                // If we already have a return value and it's different, then return None;
337                // a server can't be in two different countries.
338                if ret.is_some() && ret != Some(cc) {
339                    return None;
340                }
341
342                ret = Some(cc);
343            }
344        }
345
346        ret
347    }
348
349    /// Return the ASN the IP address is in, if this data is available.
350    pub fn lookup_asn(&self, ip: IpAddr) -> Option<u32> {
351        let cc = match ip.to_canonical() {
352            IpAddr::V4(v4) => self.map_v4.get2(&v4.into()),
353            IpAddr::V6(v6) => self.map_v6.get2(&v6.into()),
354        };
355        cc.map(|nz| nz.get())
356    }
357}
358
359/// A type that can be an address entry in one of our databases.
360trait DbAddress: FromStr {
361    /// The integer that we use to represent this kind of address.
362    type Int;
363
364    /// Convert this address to an integer.
365    fn to_int(&self) -> Self::Int;
366}
367
368impl DbAddress for u32 {
369    type Int = u32;
370
371    fn to_int(&self) -> Self::Int {
372        *self
373    }
374}
375
376impl DbAddress for Ipv6Addr {
377    type Int = u128;
378
379    fn to_int(&self) -> Self::Int {
380        (*self).into()
381    }
382}
383
384/// A line as returned by [`parse_line`].
385type ParsedLine<T> = (RangeInclusive<T>, Option<CountryCode>, Option<Asn>);
386
387/// Parse a single line from a database, expecting addresses of type T.
388///
389/// Return Ok(None) if the line is empty.
390fn parse_line<T: DbAddress>(line: &str) -> Result<Option<ParsedLine<T::Int>>, Error>
391where
392    Error: From<<T as FromStr>::Err>,
393{
394    if line.starts_with('#') {
395        return Ok(None);
396    }
397    let line = line.trim();
398    if line.is_empty() {
399        return Ok(None);
400    }
401
402    let mut split = line.split(',');
403    let from = split
404        .next()
405        .ok_or_else(|| Error::BadFormat("empty line somehow?".into()))?
406        .parse::<T>()?
407        .to_int();
408    let to = split
409        .next()
410        .ok_or_else(|| Error::BadFormat("line with insufficient commas".into()))?
411        .parse::<T>()?
412        .to_int();
413    let cc = split
414        .next()
415        .ok_or_else(|| Error::BadFormat("line with insufficient commas".into()))?;
416    let cc = match cc {
417        "" => None,
418        cc => OptionCc::from_str(cc)?.0,
419    };
420    let asn = split.next().map(|x| x.parse::<u32>()).transpose()?;
421    // Treat "0" as "no asn".
422    let asn = asn.map(NonZeroU32::try_from).transpose().ok().flatten();
423
424    Ok(Some((from..=to, cc, asn)))
425}
426
427/// A (representation of a) host on the network which may have a known country code.
428pub trait HasCountryCode {
429    /// Return the country code in which this server is most likely located.
430    ///
431    /// This is usually implemented by simple GeoIP lookup on the addresses provided by `HasAddrs`.
432    /// It follows that the server might not actually be in the returned country, but this is a
433    /// halfway decent estimate for what other servers might guess the server's location to be
434    /// (and thus useful for e.g. getting around simple geo-blocks, or having webpages return
435    /// the correct localised versions).
436    ///
437    /// Returning `None` signifies that no country code information is available. (Conflicting
438    /// GeoIP lookup results might also cause `None` to be returned.)
439    fn country_code(&self) -> Option<CountryCode>;
440}
441
442/// An export of a GeoIp database in a raw format suitable for embedding.
443///
444/// This format is deliberately undocumented, and not for other uses.
445#[cfg(feature = "export")]
446#[allow(clippy::exhaustive_structs, missing_docs)]
447pub struct RawGeoipDbExport<'a> {
448    pub ipv4_starts: &'a [u32],
449    pub ipv4_ccs: &'a [Option<CountryCode>],
450    pub ipv4_asns: Option<&'a [Option<NonZeroU32>]>,
451    pub ipv6_starts: &'a [u128],
452    pub ipv6_ccs: &'a [Option<CountryCode>],
453    pub ipv6_asns: Option<&'a [Option<NonZeroU32>]>,
454}
455
456#[cfg(feature = "export")]
457impl<'a> RawGeoipDbExport<'a> {
458    /// Save the contents of this export into a set of data files in "Path".
459    pub fn save(&self, path: &std::path::Path) -> std::io::Result<()> {
460        use std::fs::write;
461        fn into_bytes<'a, T>(data: &'a [T]) -> &'a [u8] {
462            // SAFETY: Every possible bit sequence is a valid u8.
463            let (pre, data, post) = unsafe { data.align_to::<u8>() };
464            assert!(pre.is_empty());
465            assert!(post.is_empty());
466            data
467        }
468        write(path.join("geoip_data_v4s"), into_bytes(self.ipv4_starts))?;
469        write(path.join("geoip_data_v4c"), into_bytes(self.ipv4_ccs))?;
470        if let Some(asns) = self.ipv4_asns {
471            write(path.join("geoip_data_v4a"), into_bytes(asns))?;
472        }
473        write(path.join("geoip_data_v6s"), into_bytes(self.ipv6_starts))?;
474        write(path.join("geoip_data_v6c"), into_bytes(self.ipv6_ccs))?;
475        if let Some(asns) = self.ipv6_asns {
476            write(path.join("geoip_data_v6a"), into_bytes(asns))?;
477        }
478        Ok(())
479    }
480}
481
482#[cfg(test)]
483mod test {
484    // @@ begin test lint list maintained by maint/add_warning @@
485    #![allow(clippy::bool_assert_comparison)]
486    #![allow(clippy::clone_on_copy)]
487    #![allow(clippy::dbg_macro)]
488    #![allow(clippy::mixed_attributes_style)]
489    #![allow(clippy::print_stderr)]
490    #![allow(clippy::print_stdout)]
491    #![allow(clippy::single_char_pattern)]
492    #![allow(clippy::unwrap_used)]
493    #![allow(clippy::unchecked_time_subtraction)]
494    #![allow(clippy::useless_vec)]
495    #![allow(clippy::needless_pass_by_value)]
496    #![allow(clippy::string_slice)] // See arti#2571
497    //! <!-- @@ end test lint list maintained by maint/add_warning @@ -->
498
499    use super::*;
500    use std::net::Ipv4Addr;
501
502    // NOTE(eta): this test takes a whole 1.6 seconds in *non-release* mode
503    #[test]
504    #[cfg(feature = "embedded-db")]
505    fn embedded_db() {
506        let db = GeoipDb::new_embedded();
507
508        assert_eq!(
509            db.lookup_country_code(Ipv4Addr::new(8, 8, 8, 8).into())
510                .map(|x| x.as_ref()),
511            Some("US")
512        );
513
514        assert_eq!(
515            db.lookup_country_code("2001:4860:4860::8888".parse().unwrap())
516                .map(|x| x.as_ref()),
517            Some("US")
518        );
519    }
520
521    #[test]
522    fn cc_rep() {
523        let italy = CountryCode::new("IT").unwrap();
524        assert_eq!(italy.as_ref(), "IT");
525    }
526
527    #[test]
528    fn basic_lookups() {
529        let src_v4 = r#"
530        16909056,16909311,GB,64496
531        "#;
532        let src_v6 = r#"
533        dead:beef::,dead:ffff::,??
534        fe80::,fe81::,US
535        "#;
536        let db = GeoipDb::new_from_legacy_format(src_v4, src_v6, true).unwrap();
537
538        assert_eq!(
539            db.lookup_country_code(Ipv4Addr::new(1, 2, 3, 4).into())
540                .map(|x| x.as_ref()),
541            Some("GB")
542        );
543
544        assert_eq!(
545            db.lookup_country_code(Ipv4Addr::new(1, 1, 1, 1).into()),
546            None
547        );
548
549        assert_eq!(
550            db.lookup_country_code("fe80::dead:beef".parse().unwrap())
551                .map(|x| x.as_ref()),
552            Some("US")
553        );
554
555        assert_eq!(
556            db.lookup_country_code("fe81::dead:beef".parse().unwrap()),
557            None
558        );
559        assert_eq!(
560            db.lookup_country_code("dead:beef::1".parse().unwrap()),
561            None
562        );
563
564        let mapped: IpAddr = "::ffff:1.2.3.4".parse().unwrap();
565        assert_eq!(
566            db.lookup_country_code(mapped).map(|x| x.as_ref()),
567            Some("GB")
568        );
569        assert_eq!(db.lookup_asn(mapped), Some(64496));
570
571        let compatible: IpAddr = "::1.2.3.4".parse().unwrap();
572        assert_eq!(db.lookup_country_code(compatible), None);
573    }
574
575    #[test]
576    fn cc_parse() -> Result<(), Error> {
577        // real countries.
578        assert_eq!(CountryCode::from_str("us")?, CountryCode::from_str("US")?);
579        assert_eq!(CountryCode::from_str("UY")?, CountryCode::from_str("UY")?);
580
581        // not real as of this writing, but still representable.
582        assert_eq!(CountryCode::from_str("A7")?, CountryCode::from_str("a7")?);
583        assert_eq!(CountryCode::from_str("xz")?, CountryCode::from_str("xz")?);
584
585        // Can't convert to two bytes.
586        assert!(matches!(
587            CountryCode::from_str("z"),
588            Err(Error::BadCountryCode(_))
589        ));
590        assert!(matches!(
591            CountryCode::from_str("🐻‍❄️"),
592            Err(Error::BadCountryCode(_))
593        ));
594        assert!(matches!(
595            CountryCode::from_str("Sheboygan"),
596            Err(Error::BadCountryCode(_))
597        ));
598
599        // Can convert to two bytes, but still not printable ascii
600        assert!(matches!(
601            CountryCode::from_str("\r\n"),
602            Err(Error::BadCountryCode(_))
603        ));
604        assert!(matches!(
605            CountryCode::from_str("\0\0"),
606            Err(Error::BadCountryCode(_))
607        ));
608        assert!(matches!(
609            CountryCode::from_str("¡"),
610            Err(Error::BadCountryCode(_))
611        ));
612
613        // Not a country.
614        assert!(matches!(
615            CountryCode::from_str("??"),
616            Err(Error::NowhereNotSupported)
617        ));
618
619        Ok(())
620    }
621
622    #[test]
623    fn opt_cc_parse() -> Result<(), Error> {
624        assert_eq!(
625            CountryCode::from_str("br")?,
626            OptionCc::from_str("BR")?.0.unwrap()
627        );
628        assert!(OptionCc::from_str("??")?.0.is_none());
629
630        Ok(())
631    }
632}