Skip to main content

tor_dirclient/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2#![doc = include_str!("../README.md")]
3// @@ begin lint list maintained by maint/add_warning @@
4#![allow(renamed_and_removed_lints)] // @@REMOVE_WHEN(ci_arti_stable)
5#![allow(unknown_lints)] // @@REMOVE_WHEN(ci_arti_nightly)
6#![warn(missing_docs)]
7#![warn(noop_method_call)]
8#![warn(unreachable_pub)]
9#![warn(clippy::all)]
10#![deny(clippy::await_holding_lock)]
11#![deny(clippy::cargo_common_metadata)]
12#![deny(clippy::cast_lossless)]
13#![deny(clippy::checked_conversions)]
14#![allow(clippy::cognitive_complexity)] // See arti#2556
15#![deny(clippy::debug_assert_with_mut_call)]
16#![deny(clippy::exhaustive_enums)]
17#![deny(clippy::exhaustive_structs)]
18#![deny(clippy::expl_impl_clone_on_copy)]
19#![deny(clippy::fallible_impl_from)]
20#![deny(clippy::implicit_clone)]
21#![deny(clippy::large_stack_arrays)]
22#![warn(clippy::manual_ok_or)]
23#![deny(clippy::missing_docs_in_private_items)]
24#![warn(clippy::needless_borrow)]
25#![warn(clippy::needless_pass_by_value)]
26#![warn(clippy::option_option)]
27#![warn(clippy::or_fun_call)]
28#![deny(clippy::print_stderr)]
29#![deny(clippy::print_stdout)]
30#![warn(clippy::rc_buffer)]
31#![deny(clippy::ref_option_ref)]
32#![warn(clippy::semicolon_if_nothing_returned)]
33#![warn(clippy::trait_duplication_in_bounds)]
34#![deny(clippy::unchecked_time_subtraction)]
35#![deny(clippy::unnecessary_wraps)]
36#![warn(clippy::unseparated_literal_suffix)]
37#![deny(clippy::unwrap_used)]
38#![deny(clippy::mod_module_files)]
39#![allow(clippy::let_unit_value)] // This can reasonably be done for explicitness
40#![allow(clippy::uninlined_format_args)]
41#![allow(clippy::significant_drop_in_scrutinee)] // arti/-/merge_requests/588/#note_2812945
42#![allow(clippy::result_large_err)] // temporary workaround for arti#587
43#![allow(clippy::needless_raw_string_hashes)] // complained-about code is fine, often best
44#![allow(clippy::needless_lifetimes)] // See arti#1765
45#![allow(mismatched_lifetime_syntaxes)] // temporary workaround for arti#2060
46#![allow(clippy::collapsible_if)] // See arti#2342
47#![deny(clippy::unused_async)]
48#![deny(clippy::string_slice)] // See arti#2571
49#![allow(recursion_depth_exceeding_limit)] // arti#2715, rust/issues/159228
50//! <!-- @@ end lint list maintained by maint/add_warning @@ -->
51
52// TODO probably remove this at some point - see tpo/core/arti#1060
53#![cfg_attr(
54    not(all(feature = "full", feature = "experimental")),
55    allow(unused_imports)
56)]
57
58mod body;
59mod err;
60pub mod request;
61mod response;
62mod util;
63
64use tor_circmgr::{CircMgr, DirInfo};
65use tor_error::bad_api_usage;
66use tor_rtcompat::{Runtime, SleepProvider, SleepProviderExt};
67
68// Zlib is required; the others are optional.
69#[cfg(feature = "xz")]
70use async_compression::futures::bufread::XzDecoder;
71use async_compression::futures::bufread::ZlibDecoder;
72#[cfg(feature = "zstd")]
73use async_compression::futures::bufread::ZstdDecoder;
74
75use futures::FutureExt;
76use futures::io::{
77    AsyncBufRead, AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader,
78};
79use std::sync::Arc;
80use std::time::Duration;
81use tracing::{info, instrument};
82
83pub use err::{Error, RequestError, RequestFailedError};
84pub use response::{DirResponse, SourceInfo};
85
86/// Type for results returned in this crate.
87pub type Result<T> = std::result::Result<T, Error>;
88
89/// Type for internal results  containing a RequestError.
90pub type RequestResult<T> = std::result::Result<T, RequestError>;
91
92/// Flag to declare whether a request is always anonymized or not.
93///
94/// This is used by tor-dirclient to control whether *other* deanonymizing metadata
95/// might be added to the request (eg in request headers):
96/// Some requests (like those to download onion service descriptors) are always
97/// anonymized, and should never be sent in a way that leaks information about
98/// our settings or configuration.
99///
100/// It is up to the *caller* of `tor-dirclient` to ensure that
101///
102///   - every request whose anonymization status is `AnonymizedRequest::Direct`
103///     is sent only over non-anonymous connections.
104///
105///     (Sending an `AnonymizedRequest::Direct` request over an anonymized connection
106///     would weaken the connection's anonymity, and can therefore weaken the anonymity
107///     of user traffic sharing the same circuit.)
108///
109///   - every request whose anonymization status is `AnonymizedRequest::Anonymized`
110///     is sent over only anonymous connections (ie, multi-hop circuits).
111///
112///     (Sending an `AnonymizedRequest::Anonymized` request over a direct connection
113///     would directly reveal user behaviour data to the directory server.)
114///
115/// TODO the calling code cannot easily be sure to get this right this because
116/// the anonymization status is a run-time property and the choice of connection kind
117/// is statically defined in the calling code.  (Perhaps this could be checked in tests?)
118#[derive(Copy, Clone, Debug, Eq, PartialEq)]
119#[non_exhaustive]
120pub enum AnonymizedRequest {
121    /// This request's content or semantics reveals or is correlated with sensitive information.
122    ///
123    /// For example, requests for hidden service descriptors reveal which hidden services
124    /// the client is connecting to.
125    ///
126    /// The request must be sent over an anonymous circuit by the caller
127    /// and no additional deanonymizing information should be added to it by `tor-dirclient`.
128    /// (For example, no client-version-specific information should be
129    /// sent in HTTP headers when the request is made.)
130    Anonymized,
131
132    /// Making this request does not reveal anything sensitive, nor any user behaviour.
133    ///
134    /// The request body is uncorrelated with such things as the websites the user might visit,
135    /// the onion services the user is visiting or running, etc.
136    ///
137    /// For example, requests for all router microdescriptors are made by all clients,
138    /// so which microdescriptor(s) are requested reveals nothing to any attacker.
139    ///
140    /// tor-dirclient is allowed to add include information about our capabilities
141    /// when sending this request.
142    /// The request must *not* be sent over an anonymous circuit by the caller
143    /// (at least, not one used for anything else).
144    Direct,
145}
146
147/// Fetch the resource described by `req` over the Tor network.
148///
149/// Circuits are built or found using `circ_mgr`, using paths
150/// constructed using `dirinfo`.
151///
152/// For more fine-grained control over the circuit and stream used,
153/// construct them yourself, and then call [`send_request`] instead.
154///
155/// # TODO
156///
157/// This is the only function in this crate that knows about CircMgr and
158/// DirInfo.  Perhaps this function should move up a level into DirMgr?
159#[instrument(level = "trace", skip_all)]
160pub async fn get_resource<CR, R, SP>(
161    req: &CR,
162    dirinfo: DirInfo<'_>,
163    runtime: &SP,
164    circ_mgr: Arc<CircMgr<R>>,
165) -> Result<DirResponse>
166where
167    CR: request::Requestable + ?Sized,
168    R: Runtime,
169    SP: SleepProvider,
170{
171    let tunnel = circ_mgr.get_or_launch_dir(dirinfo).await?;
172
173    if req.anonymized() == AnonymizedRequest::Anonymized {
174        return Err(bad_api_usage!("Tried to use get_resource for an anonymized request").into());
175    }
176
177    // TODO(nickm) This should be an option, and is too long.
178    let begin_timeout = Duration::from_secs(5);
179    let source = match SourceInfo::from_tunnel(&tunnel) {
180        Ok(source) => source,
181        Err(e) => {
182            return Err(Error::RequestFailed(RequestFailedError {
183                source: None,
184                error: e.into(),
185            }));
186        }
187    };
188
189    let wrap_err = |error| {
190        Error::RequestFailed(RequestFailedError {
191            source: source.clone(),
192            error,
193        })
194    };
195
196    req.check_circuit_skip_optimization(&tunnel)
197        .await
198        .map_err(wrap_err)?;
199
200    // Launch the stream.
201    let mut stream = runtime
202        .timeout(begin_timeout, tunnel.begin_dir_stream())
203        .await
204        .map_err(RequestError::from)
205        .map_err(wrap_err)?
206        .map_err(RequestError::from)
207        .map_err(wrap_err)?; // TODO(nickm) handle fatalities here too
208
209    // TODO: Perhaps we want separate timeouts for each phase of this.
210    // For now, we just use higher-level timeouts in `dirmgr`.
211    let r = send_request(runtime, req, &mut stream, source.clone()).await;
212
213    if should_retire_circ(&r) {
214        retire_circ(&circ_mgr, &tunnel.unique_id(), "Partial response");
215    }
216
217    r
218}
219
220/// Return true if `result` holds an error indicating that we should retire the
221/// circuit used for the corresponding request.
222fn should_retire_circ(result: &Result<DirResponse>) -> bool {
223    match result {
224        Err(e) => e.should_retire_circ(),
225        Ok(dr) => dr.error().map(RequestError::should_retire_circ) == Some(true),
226    }
227}
228
229/// Fetch a Tor directory object from a provided stream.
230#[deprecated(since = "0.8.1", note = "Use send_request instead.")]
231pub async fn download<R, S, SP>(
232    runtime: &SP,
233    req: &R,
234    stream: &mut S,
235    source: Option<SourceInfo>,
236) -> Result<DirResponse>
237where
238    R: request::Requestable + ?Sized,
239    S: AsyncRead + AsyncWrite + Send + Unpin,
240    SP: SleepProvider,
241{
242    send_request(runtime, req, stream, source).await
243}
244
245/// Fetch or upload a Tor directory object using the provided stream.
246///
247/// To do this, we send a simple HTTP/1.0 request for the described
248/// object in `req` over `stream`, and then wait for a response.  In
249/// log messages, we describe the origin of the data as coming from
250/// `source`.
251///
252/// # Notes
253///
254/// It's kind of bogus to have a 'source' field here at all; we may
255/// eventually want to remove it.
256///
257/// This function doesn't close the stream; you may want to do that
258/// yourself.
259///
260/// The only error variant returned is [`Error::RequestFailed`].
261// TODO: should the error return type change to `RequestFailedError`?
262// If so, that would simplify some code in_dirmgr::bridgedesc.
263pub async fn send_request<R, S, SP>(
264    runtime: &SP,
265    req: &R,
266    stream: &mut S,
267    source: Option<SourceInfo>,
268) -> Result<DirResponse>
269where
270    R: request::Requestable + ?Sized,
271    S: AsyncRead + AsyncWrite + Send + Unpin,
272    SP: SleepProvider,
273{
274    let wrap_err = |error| {
275        Error::RequestFailed(RequestFailedError {
276            source: source.clone(),
277            error,
278        })
279    };
280
281    let partial_ok = req.partial_response_body_ok();
282    let maxlen = req.max_response_len();
283    let anonymized = req.anonymized();
284    let req = req.make_request().map_err(wrap_err)?;
285    let method = req.method().clone();
286    let encoded = util::encode_request(&req);
287
288    // Write the request.
289    for chunk in encoded.iter() {
290        stream
291            .write_all(chunk)
292            .await
293            .map_err(RequestError::from)
294            .map_err(wrap_err)?;
295    }
296    stream
297        .flush()
298        .await
299        .map_err(RequestError::from)
300        .map_err(wrap_err)?;
301
302    let mut buffered = BufReader::new(stream);
303
304    // Handle the response
305    let header = read_headers(runtime, &mut buffered)
306        .await
307        .map_err(wrap_err)?;
308    if header.status != Some(200) {
309        return Ok(DirResponse::new(
310            method,
311            header.status.unwrap_or(0),
312            header.status_message,
313            None,
314            vec![],
315            source,
316        ));
317    }
318
319    let mut decoder =
320        get_decoder(buffered, header.encoding.as_deref(), anonymized).map_err(wrap_err)?;
321
322    let mut result = Vec::new();
323    let ok = read_and_decompress(runtime, &mut decoder, maxlen, &mut result).await;
324
325    let ok = match (partial_ok, ok, result.len()) {
326        (true, Err(e), n) if n > 0 => {
327            // Note that we _don't_ return here: we want the partial response.
328            Err(e)
329        }
330        (_, Err(e), _) => {
331            return Err(wrap_err(e));
332        }
333        (_, Ok(()), _) => Ok(()),
334    };
335
336    Ok(DirResponse::new(
337        method,
338        200,
339        None,
340        ok.err(),
341        result,
342        source,
343    ))
344}
345
346/// Maximum length for the HTTP headers in a single request or response.
347///
348/// Chosen more or less arbitrarily.
349const MAX_HEADERS_LEN: usize = 16384;
350
351/// Read and parse HTTP/1 headers from `stream`.
352///
353/// Wraps around [`read_headers_internal()`] with a timeout.
354async fn read_headers<S, SP>(runtime: &SP, stream: &mut S) -> RequestResult<HeaderStatus>
355where
356    S: AsyncBufRead + Unpin,
357    SP: SleepProvider,
358{
359    // Just like with the other timeouts in this code, this should probably be
360    // configurable and may even be too much.
361    let read_timeout = Duration::from_secs(10);
362    runtime
363        .timeout(read_timeout, read_headers_internal(stream))
364        .await
365        .map_err(|_| RequestError::DirTimeout)
366        .flatten()
367}
368
369/// Internal version of [`read_headers()`] without a timeout.
370async fn read_headers_internal<S>(stream: &mut S) -> RequestResult<HeaderStatus>
371where
372    S: AsyncBufRead + Unpin,
373{
374    let mut buf = Vec::with_capacity(1024);
375
376    // Note: Using `take` here will ensure that we will get an EOF if the header length
377    // would exceed the maximum.  No extra bytes will actually be extracted.
378    let mut limited_stream = stream.take(MAX_HEADERS_LEN as u64);
379
380    loop {
381        // TODO: it's inefficient to do this a line at a time; it would
382        // probably be better to read until the CRLF CRLF ending of the
383        // response.  But this should be fast enough.
384        let n = limited_stream.read_until(b'\n', &mut buf).await?;
385
386        // TODO(nickm): Better maximum and/or let this expand.
387        let mut headers = [httparse::EMPTY_HEADER; 32];
388        let mut response = httparse::Response::new(&mut headers);
389
390        match response.parse(&buf[..])? {
391            httparse::Status::Partial => {
392                // We didn't get a whole response; we may need to try again.
393
394                if n == 0 {
395                    // We hit an EOF; no more progress can be made.
396                    return Err(RequestError::TruncatedHeaders);
397                }
398
399                if buf.len() >= MAX_HEADERS_LEN {
400                    return Err(RequestError::HeadersTooLong(buf.len()));
401                }
402            }
403            httparse::Status::Complete(n_parsed) => {
404                if response.code != Some(200) {
405                    return Ok(HeaderStatus {
406                        status: response.code,
407                        status_message: response.reason.map(str::to_owned),
408                        encoding: None,
409                    });
410                }
411                let encoding = if let Some(enc) = response
412                    .headers
413                    .iter()
414                    .find(|h| h.name == "Content-Encoding")
415                {
416                    Some(String::from_utf8(enc.value.to_vec())?)
417                } else {
418                    None
419                };
420                /*
421                if let Some(clen) = response.headers.iter().find(|h| h.name == "Content-Length") {
422                    let clen = std::str::from_utf8(clen.value)?;
423                    length = Some(clen.parse()?);
424                }
425                 */
426                assert!(n_parsed == buf.len());
427                return Ok(HeaderStatus {
428                    status: Some(200),
429                    status_message: None,
430                    encoding,
431                });
432            }
433        }
434        if n == 0 {
435            return Err(RequestError::TruncatedHeaders);
436        }
437    }
438}
439
440/// Return value from read_headers
441#[derive(Debug, Clone)]
442struct HeaderStatus {
443    /// HTTP status code.
444    status: Option<u16>,
445    /// HTTP status message associated with the status code.
446    status_message: Option<String>,
447    /// The Content-Encoding header, if any.
448    encoding: Option<String>,
449}
450
451/// Helper: download directory information from `stream` and
452/// decompress it into a result buffer.  Assumes that `buf` is empty.
453///
454/// If we get more than maxlen bytes after decompression, give an error.
455///
456/// Returns the status of our download attempt, stores any data that
457/// we were able to download into `result`.  Existing contents of
458/// `result` are overwritten.
459async fn read_and_decompress<S, SP>(
460    runtime: &SP,
461    mut stream: S,
462    maxlen: usize,
463    result: &mut Vec<u8>,
464) -> RequestResult<()>
465where
466    S: AsyncRead + Unpin,
467    SP: SleepProvider,
468{
469    let buffer_window_size = 1024;
470    let mut written_total: usize = 0;
471    // TODO(cve): This should be configurable.
472    // Originally, this was set to 10s, however we have bumped it to 60s for
473    // the following reasons:
474    // 1. The largest possible netdoc to retrieve is /tor/server/all, which is
475    //    roughly 30MB = 240 Megabit with an average gzip compression applied.
476    // 2. Downloading this before the 10s timeout completes would require an
477    //    internet bandwidth of at least 24Mbit/s.
478    // 3. According to Wikipedia (Nov. 2026), there are at least 17 countries
479    //    where the average internet speed is below 24Mbit/s.  Besides, there
480    //    may be other conditions which might make the connection slower, such
481    //    as a DoS, sharing a coaxial cable with neighbors, etc.
482    // 4. Assuming a speed of 5Mbit/s, downloading /tor/server/all compressed
483    //    would take 48s.  In order to be a bit more tolerant, we round it up
484    //    to 60s.
485    //
486    // I am aware that these estimates are probably unrealistic because
487    // downloading all plain descriptors is an edge-case that almost no client
488    // uses.  However, it should still be a reasonable upper bound, given that
489    // certain bridges may even have a bandwidth as low as 1Mbit/s.
490    //
491    // For directory mirrors and authorities, this should not be a problem,
492    // because they do not download the descriptors in bulk but rather
493    // individually across multiple requests or potentially not at all, in the
494    // case of an authority, which may even get the descriptor POSTed to them.
495    let read_timeout = Duration::from_secs(60);
496    let timer = runtime.sleep(read_timeout).fuse();
497    futures::pin_mut!(timer);
498
499    loop {
500        // allocate buffer for next read
501        result.resize(written_total + buffer_window_size, 0);
502        let buf: &mut [u8] = &mut result[written_total..written_total + buffer_window_size];
503
504        let status = futures::select! {
505            status = stream.read(buf).fuse() => status,
506            _ = timer => {
507                result.resize(written_total, 0); // truncate as needed
508                return Err(RequestError::DirTimeout);
509            }
510        };
511        let written_in_this_loop = match status {
512            Ok(n) => n,
513            Err(other) => {
514                result.resize(written_total, 0); // truncate as needed
515                return Err(other.into());
516            }
517        };
518
519        written_total += written_in_this_loop;
520
521        // exit conditions below
522
523        if written_in_this_loop == 0 {
524            /*
525            in case we read less than `buffer_window_size` in last `read`
526            we need to shrink result because otherwise we'll return those
527            un-read 0s
528            */
529            if written_total < result.len() {
530                result.resize(written_total, 0);
531            }
532            return Ok(());
533        }
534
535        // TODO: It would be good to detect compression bombs, but
536        // that would require access to the internal stream, which
537        // would in turn require some tricky programming.  For now, we
538        // use the maximum length here to prevent an attacker from
539        // filling our RAM.
540        if written_total > maxlen {
541            result.resize(maxlen, 0);
542            return Err(RequestError::ResponseTooLong(written_total));
543        }
544    }
545}
546
547/// Retire a directory circuit because of an error we've encountered on it.
548fn retire_circ<R>(circ_mgr: &Arc<CircMgr<R>>, id: &tor_proto::circuit::UniqId, error: &str)
549where
550    R: Runtime,
551{
552    info!(
553        "{}: Retiring circuit because of directory failure: {}",
554        &id, &error
555    );
556    circ_mgr.retire_circ(id);
557}
558
559/// Memory limit of the LZMA dictionary we are willing to allocate.
560///
561/// Without it, it may lead to an allocation of 4GiB, which is a bit overkill.
562///
563/// Right now, we use 16 MiB, as the spec limits the compression quality to
564/// "6", meaning 9 MiB of decompression memory, which rounds up to 16 MiB in
565/// terms of base 2.
566///
567/// From the xz(1) manual page:
568/// ```text
569/// Preset   DictSize   CompCPU   CompMem   DecMem
570///  -0e     256 KiB       8        4 MiB    1 MiB
571///  -1e       1 MiB       8       13 MiB    2 MiB
572///  -2e       2 MiB       8       25 MiB    3 MiB
573///  -3e       4 MiB       7       48 MiB    5 MiB
574///  -4e       4 MiB       8       48 MiB    5 MiB
575///  -5e       8 MiB       7       94 MiB    9 MiB
576///  -6e       8 MiB       8       94 MiB    9 MiB
577///  -7e      16 MiB       8      186 MiB   17 MiB
578///  -8e      32 MiB       8      370 MiB   33 MiB
579///  -9e      64 MiB       8      674 MiB   65 MiB
580/// ```
581const LZMA_DICT_MEM_LIMIT: u64 = 1 << 24; // 16 MiB
582
583/// Helper: Return a boxed decoder object that wraps the stream  $s.
584macro_rules! decoder {
585    (XzDecoder, $s:expr) => {{
586        let mut decoder = XzDecoder::with_mem_limit($s, LZMA_DICT_MEM_LIMIT);
587        decoder.multiple_members(true);
588        Ok(Box::new(decoder))
589    }};
590    ($dec:ident, $s:expr) => {{
591        let mut decoder = $dec::new($s);
592        decoder.multiple_members(true);
593        Ok(Box::new(decoder))
594    }};
595}
596
597/// Wrap `stream` in an appropriate type to undo the content encoding
598/// as described in `encoding`.
599fn get_decoder<'a, S: AsyncBufRead + Unpin + Send + 'a>(
600    stream: S,
601    encoding: Option<&str>,
602    anonymized: AnonymizedRequest,
603) -> RequestResult<Box<dyn AsyncRead + Unpin + Send + 'a>> {
604    use AnonymizedRequest::Direct;
605    match (encoding, anonymized) {
606        (None | Some("identity"), _) => Ok(Box::new(stream)),
607        (Some("deflate"), _) => decoder!(ZlibDecoder, stream),
608        // We only admit to supporting these on a direct connection; otherwise,
609        // a hostile directory could send them back even though we hadn't
610        // requested them.
611        #[cfg(feature = "xz")]
612        (Some("x-tor-lzma"), Direct) => decoder!(XzDecoder, stream),
613        #[cfg(feature = "zstd")]
614        (Some("x-zstd"), Direct) => decoder!(ZstdDecoder, stream),
615        (Some(other), _) => Err(RequestError::ContentEncoding(other.into())),
616    }
617}
618
619#[cfg(test)]
620mod test {
621    // @@ begin test lint list maintained by maint/add_warning @@
622    #![allow(clippy::bool_assert_comparison)]
623    #![allow(clippy::clone_on_copy)]
624    #![allow(clippy::dbg_macro)]
625    #![allow(clippy::mixed_attributes_style)]
626    #![allow(clippy::print_stderr)]
627    #![allow(clippy::print_stdout)]
628    #![allow(clippy::single_char_pattern)]
629    #![allow(clippy::unwrap_used)]
630    #![allow(clippy::unchecked_time_subtraction)]
631    #![allow(clippy::useless_vec)]
632    #![allow(clippy::needless_pass_by_value)]
633    #![allow(clippy::string_slice)] // See arti#2571
634    //! <!-- @@ end test lint list maintained by maint/add_warning @@ -->
635    use super::*;
636    use tor_rtmock::io::stream_pair;
637
638    use tor_rtmock::simple_time::SimpleMockTimeProvider;
639    use web_time_compat::{SystemTime, SystemTimeExt};
640
641    use futures_await_test::async_test;
642
643    // Basic decompression wrapper.
644    async fn decomp_basic(
645        encoding: Option<&str>,
646        data: &[u8],
647        maxlen: usize,
648    ) -> (RequestResult<()>, Vec<u8>) {
649        // We don't need to do anything fancy here, since we aren't simulating
650        // a timeout.
651        #[allow(deprecated)] // TODO #1885
652        let mock_time = SimpleMockTimeProvider::from_wallclock(SystemTime::get());
653
654        let mut output = Vec::new();
655        let mut stream = match get_decoder(data, encoding, AnonymizedRequest::Direct) {
656            Ok(s) => s,
657            Err(e) => return (Err(e), output),
658        };
659
660        let r = read_and_decompress(&mock_time, &mut stream, maxlen, &mut output).await;
661
662        (r, output)
663    }
664
665    #[async_test]
666    async fn decompress_identity() -> RequestResult<()> {
667        let mut text = Vec::new();
668        for _ in 0..1000 {
669            text.extend(b"This is a string with a nontrivial length that we'll use to make sure that the loop is executed more than once.");
670        }
671
672        let limit = 10 << 20;
673        let (s, r) = decomp_basic(None, &text[..], limit).await;
674        s?;
675        assert_eq!(r, text);
676
677        let (s, r) = decomp_basic(Some("identity"), &text[..], limit).await;
678        s?;
679        assert_eq!(r, text);
680
681        // Try truncated result
682        let limit = 100;
683        let (s, r) = decomp_basic(Some("identity"), &text[..], limit).await;
684        assert!(s.is_err());
685        assert_eq!(r, &text[..100]);
686
687        Ok(())
688    }
689
690    #[async_test]
691    async fn decomp_zlib() -> RequestResult<()> {
692        let compressed =
693            hex::decode("789cf3cf4b5548cb2cce500829cf8730825253200ca79c52881c00e5970c88").unwrap();
694
695        let limit = 10 << 20;
696        let (s, r) = decomp_basic(Some("deflate"), &compressed, limit).await;
697        s?;
698        assert_eq!(r, b"One fish Two fish Red fish Blue fish");
699
700        Ok(())
701    }
702
703    #[cfg(feature = "zstd")]
704    #[async_test]
705    async fn decomp_zstd() -> RequestResult<()> {
706        let compressed = hex::decode("28b52ffd24250d0100c84f6e6520666973682054776f526564426c756520666973680a0200600c0e2509478352cb").unwrap();
707        let limit = 10 << 20;
708        let (s, r) = decomp_basic(Some("x-zstd"), &compressed, limit).await;
709        s?;
710        assert_eq!(r, b"One fish Two fish Red fish Blue fish\n");
711
712        Ok(())
713    }
714
715    #[cfg(feature = "xz")]
716    #[async_test]
717    async fn decomp_xz() -> RequestResult<()> {
718        // echo "One fish Two fish Red fish Blue fish" | xz -6 | xxd -p
719        let compressed = hex::decode("fd377a585a000004e6d6b44604c02525210116000000000000000000bfdbca33e00024001d5d00279b88a202ca8612cfb3c19c87c34248a570451e4851d3323d34ab8000000000000901af64854c91f600014125281fe0821fb6f37d010000000004595a").unwrap();
720        let limit = LZMA_DICT_MEM_LIMIT as usize;
721        let (s, r) = decomp_basic(Some("x-tor-lzma"), &compressed, limit).await;
722        s?;
723        assert_eq!(r, b"One fish Two fish Red fish Blue fish\n");
724
725        Ok(())
726    }
727
728    #[cfg(feature = "xz")]
729    #[async_test]
730    async fn decomp_xz_bad() -> RequestResult<()> {
731        // Not so good at tiny files...
732        let compressed = hex::decode("fd377a585a000004e6d6b446020021011c00000010cf58cce00024001d5d00279b88a202ca8612cfb3c19c87c34248a570451e4851d3323d34ab8000000000000901af64854c91f600013925d6ec06651fb6f37d010000000004595a").unwrap();
733        let limit = 10 << 20;
734        let (s, _) = decomp_basic(Some("x-tor-lzma"), &compressed, limit).await;
735        assert!(matches!(s, Err(RequestError::IoError(_))));
736
737        Ok(())
738    }
739
740    #[async_test]
741    async fn decomp_unknown() {
742        let compressed = hex::decode("28b52ffd24250d0100c84f6e6520666973682054776f526564426c756520666973680a0200600c0e2509478352cb").unwrap();
743        let limit = 10 << 20;
744        let (s, _r) = decomp_basic(Some("x-proprietary-rle"), &compressed, limit).await;
745
746        assert!(matches!(s, Err(RequestError::ContentEncoding(_))));
747    }
748
749    #[async_test]
750    async fn decomp_bad_data() {
751        let compressed = b"This is not good zlib data";
752        let limit = 10 << 20;
753        let (s, _r) = decomp_basic(Some("deflate"), compressed, limit).await;
754
755        // This should possibly be a different type in the future.
756        assert!(matches!(s, Err(RequestError::IoError(_))));
757    }
758
759    #[test]
760    fn headers_ok() -> RequestResult<()> {
761        let text = b"HTTP/1.0 200 OK\r\nDate: ignored\r\nContent-Encoding: Waffles\r\n\r\n";
762
763        let mut s = &text[..];
764        let h = tor_rtcompat::test_with_one_runtime!(async |rt| read_headers(&rt, &mut s).await)?;
765
766        assert_eq!(h.status, Some(200));
767        assert_eq!(h.encoding.as_deref(), Some("Waffles"));
768
769        // now try truncated
770        let mut s = &text[..15];
771        let h = tor_rtcompat::test_with_one_runtime!(async |rt| read_headers(&rt, &mut s).await);
772        assert!(matches!(h, Err(RequestError::TruncatedHeaders)));
773
774        // now try with no encoding.
775        let text = b"HTTP/1.0 404 Not found\r\n\r\n";
776        let mut s = &text[..];
777        let h = tor_rtcompat::test_with_one_runtime!(async |rt| read_headers(&rt, &mut s).await)?;
778
779        assert_eq!(h.status, Some(404));
780        assert!(h.encoding.is_none());
781
782        Ok(())
783    }
784
785    #[test]
786    fn headers_bogus() {
787        let text = b"HTTP/999.0 WHAT EVEN\r\n\r\n";
788        let mut s = &text[..];
789        let h = tor_rtcompat::test_with_one_runtime!(async |rt| read_headers(&rt, &mut s).await);
790
791        assert!(h.is_err());
792        assert!(matches!(h, Err(RequestError::HttparseError(_))));
793    }
794
795    /// Run a trivial download example with a response provided as a binary
796    /// string.
797    ///
798    /// Return the directory response (if any) and the request as encoded (if
799    /// any.)
800    fn run_download_test<Req: request::Requestable>(
801        req: Req,
802        response: &[u8],
803    ) -> (Result<DirResponse>, RequestResult<Vec<u8>>) {
804        let (mut s1, s2) = stream_pair();
805        let (mut s2_r, mut s2_w) = s2.split();
806
807        tor_rtcompat::test_with_one_runtime!(|rt| async move {
808            let rt2 = rt.clone();
809            let (v1, v2, v3): (
810                Result<DirResponse>,
811                RequestResult<Vec<u8>>,
812                RequestResult<()>,
813            ) = futures::join!(
814                async {
815                    // Run the download function.
816                    let r = send_request(&rt, &req, &mut s1, None).await;
817                    s1.close().await.map_err(|error| {
818                        Error::RequestFailed(RequestFailedError {
819                            source: None,
820                            error: error.into(),
821                        })
822                    })?;
823                    r
824                },
825                async {
826                    // Take the request from the client, and return it in "v2"
827                    let mut v = Vec::new();
828                    s2_r.read_to_end(&mut v).await?;
829                    Ok(v)
830                },
831                async {
832                    // Send back a response.
833                    s2_w.write_all(response).await?;
834                    // We wait a moment to give the other side time to notice it
835                    // has data.
836                    //
837                    // (Tentative diagnosis: The `async-compress` crate seems to
838                    // be behave differently depending on whether the "close"
839                    // comes right after the incomplete data or whether it comes
840                    // after a delay.  If there's a delay, it notices the
841                    // truncated data and tells us about it. But when there's
842                    // _no_delay, it treats the data as an error and doesn't
843                    // tell our code.)
844
845                    // TODO: sleeping in tests is not great.
846                    rt2.sleep(Duration::from_millis(50)).await;
847                    s2_w.close().await?;
848                    Ok(())
849                }
850            );
851
852            assert!(v3.is_ok());
853
854            (v1, v2)
855        })
856    }
857
858    #[test]
859    fn test_send_request() -> RequestResult<()> {
860        let req: request::MicrodescRequest = vec![[9; 32]].into_iter().collect();
861
862        let (response, request) = run_download_test(
863            req,
864            b"HTTP/1.0 200 OK\r\n\r\nThis is where the descs would go.",
865        );
866
867        let request = request?;
868        assert!(request[..].starts_with(
869            b"GET /tor/micro/d/CQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQk HTTP/1.0\r\n"
870        ));
871
872        let response = response.unwrap();
873        assert_eq!(response.status_code(), 200);
874        assert!(!response.is_partial());
875        assert!(response.error().is_none());
876        assert!(response.source().is_none());
877        let out_ref = response.output_unchecked();
878        assert_eq!(out_ref, b"This is where the descs would go.");
879        let out = response.into_output_unchecked();
880        assert_eq!(&out, b"This is where the descs would go.");
881
882        Ok(())
883    }
884
885    #[test]
886    fn test_download_truncated() {
887        // Request only one md, so "partial ok" will not be set.
888        let req: request::MicrodescRequest = vec![[9; 32]].into_iter().collect();
889        let mut response_text: Vec<u8> =
890            (*b"HTTP/1.0 200 OK\r\nContent-Encoding: deflate\r\n\r\n").into();
891        // "One fish two fish" as above twice, but truncated the second time
892        response_text.extend(
893            hex::decode("789cf3cf4b5548cb2cce500829cf8730825253200ca79c52881c00e5970c88").unwrap(),
894        );
895        response_text.extend(
896            hex::decode("789cf3cf4b5548cb2cce500829cf8730825253200ca79c52881c00e5").unwrap(),
897        );
898        let (response, request) = run_download_test(req, &response_text);
899        assert!(request.is_ok());
900        assert!(response.is_err()); // The whole download should fail, since partial_ok wasn't set.
901
902        // request two microdescs, so "partial_ok" will be set.
903        let req: request::MicrodescRequest = vec![[9; 32]; 2].into_iter().collect();
904
905        let (response, request) = run_download_test(req, &response_text);
906        assert!(request.is_ok());
907
908        let response = response.unwrap();
909        assert_eq!(response.status_code(), 200);
910        assert!(response.error().is_some());
911        assert!(response.is_partial());
912        assert!(response.output_unchecked().len() < 37 * 2);
913        assert!(response.output_unchecked().starts_with(b"One fish"));
914    }
915
916    #[test]
917    fn test_404() {
918        let req: request::MicrodescRequest = vec![[9; 32]].into_iter().collect();
919        let response_text = b"HTTP/1.0 418 I'm a teapot\r\n\r\n";
920        let (response, _request) = run_download_test(req, response_text);
921
922        assert_eq!(response.unwrap().status_code(), 418);
923    }
924
925    #[test]
926    fn test_headers_truncated() {
927        let req: request::MicrodescRequest = vec![[9; 32]].into_iter().collect();
928        let response_text = b"HTTP/1.0 404 truncation happens here\r\n";
929        let (response, _request) = run_download_test(req, response_text);
930
931        assert!(matches!(
932            response,
933            Err(Error::RequestFailed(RequestFailedError {
934                error: RequestError::TruncatedHeaders,
935                ..
936            }))
937        ));
938
939        // Try a completely empty response.
940        let req: request::MicrodescRequest = vec![[9; 32]].into_iter().collect();
941        let response_text = b"";
942        let (response, _request) = run_download_test(req, response_text);
943
944        assert!(matches!(
945            response,
946            Err(Error::RequestFailed(RequestFailedError {
947                error: RequestError::TruncatedHeaders,
948                ..
949            }))
950        ));
951    }
952
953    #[test]
954    fn test_headers_too_long() {
955        let req: request::MicrodescRequest = vec![[9; 32]].into_iter().collect();
956        let mut response_text: Vec<u8> = (*b"HTTP/1.0 418 I'm a teapot\r\nX-Too-Many-As: ").into();
957        response_text.resize(16384, b'A');
958        let (response, _request) = run_download_test(req, &response_text);
959
960        assert!(response.as_ref().unwrap_err().should_retire_circ());
961        assert!(matches!(
962            response,
963            Err(Error::RequestFailed(RequestFailedError {
964                error: RequestError::HeadersTooLong(_),
965                ..
966            }))
967        ));
968    }
969
970    #[test]
971    fn test_bad_utf8() {
972        let req: request::MicrodescRequest = vec![[9; 32]].into_iter().collect();
973        let faulty_utf8 = vec![0, 159, 146, 150];
974
975        let mut response_text: Vec<u8> = b"HTTP/1.0 200 OK\r\n\r\n".into();
976        response_text.extend(faulty_utf8);
977
978        let (response, _request) = run_download_test(req, &response_text);
979
980        assert!(matches!(
981            response.unwrap().into_output_string().unwrap_err(),
982            RequestFailedError {
983                error: RequestError::Utf8Encoding(_),
984                ..
985            }
986        ));
987    }
988}