Skip to main content

tor_netdoc/parse/
tokenize.rs

1//! Break a string into a set of directory-object Items.
2//!
3//! This module defines Item, which represents a basic entry in a
4//! directory document, and NetDocReader, which is used to break a
5//! string into Items.
6
7use crate::parse::keyword::Keyword;
8use crate::types::misc::FromBytes;
9use crate::util::PeekableIterator;
10use crate::{Error, NetdocErrorKind as EK, Pos, Result};
11use base64ct::{Base64, Encoding};
12use itertools::Itertools;
13use std::cell::{Ref, RefCell};
14use std::iter::Peekable;
15use std::str::FromStr;
16use tor_error::internal;
17
18/// Useful constants for netdoc object syntax
19pub(crate) mod object {
20    /// indicates the start of an object
21    pub(crate) const BEGIN_STR: &str = "-----BEGIN ";
22    /// indicates the end of an object
23    pub(crate) const END_STR: &str = "-----END ";
24    /// indicates the end of a begin or end tag.
25    pub(crate) const TAG_END: &str = "-----";
26    /// Maximum PEM base64 line length (not enforced during parsing)
27    pub(crate) const BASE64_PEM_MAX_LINE: usize = 64;
28}
29
30/// Return true iff a given character is "space" according to the rules
31/// of dir-spec.txt
32pub(crate) fn is_sp(c: char) -> bool {
33    c == ' ' || c == '\t'
34}
35/// Check that all the characters in `s` are valid base64.
36///
37/// This is not a perfect check for base64ness -- it is mainly meant
38/// to help us recover after unterminated base64.
39fn b64check(s: &str) -> Result<()> {
40    for b in s.bytes() {
41        match b {
42            b'=' => (),
43            b'a'..=b'z' => (),
44            b'A'..=b'Z' => (),
45            b'0'..=b'9' => (),
46            b'/' | b'+' => (),
47            _ => {
48                return Err(EK::BadObjectBase64.at_pos(Pos::at(s)));
49            }
50        };
51    }
52    Ok(())
53}
54
55/// A tagged object that is part of a directory Item.
56///
57/// This represents a single blob within a pair of "-----BEGIN
58/// FOO-----" and "-----END FOO-----".  The data is not guaranteed to
59/// be actual base64 when this object is created: doing so would
60/// require either that we parse the base64 twice, or that we allocate
61/// a buffer to hold the data before it's needed.
62#[derive(Clone, Copy, Debug)]
63pub(crate) struct Object<'a> {
64    /// Reference to the "tag" string (the 'foo') in the BEGIN line.
65    tag: &'a str,
66    /// Reference to the allegedly base64-encoded data.  This may or
67    /// may not actually be base64 at this point.
68    data: &'a str,
69    /// Reference to the END line for this object.  This doesn't
70    /// need to be parsed, but it's used to find where this object
71    /// ends.
72    endline: &'a str,
73}
74
75/// A single part of a directory object.
76///
77/// Each Item -- called an "entry" in dir-spec.txt -- has a keyword, a
78/// (possibly empty) set of arguments, and an optional object.
79///
80/// This is a zero-copy implementation that points to slices within a
81/// containing string.
82#[derive(Clone, Debug)]
83pub(crate) struct Item<'a, K: Keyword> {
84    /// The keyword that determines the type of this item.
85    kwd: K,
86    /// A reference to the actual string that defines the keyword for
87    /// this item.
88    kwd_str: &'a str,
89    /// Reference to the arguments that appear in the same line after the
90    /// keyword.  Does not include the terminating newline or the
91    /// space that separates the keyword for its arguments.
92    args: &'a str,
93    /// The arguments, split by whitespace.  This vector is constructed
94    /// as needed, using interior mutability.
95    split_args: RefCell<Option<Vec<&'a str>>>,
96    /// If present, a base-64-encoded object that appeared at the end
97    /// of this item.
98    object: Option<Object<'a>>,
99}
100
101/// A cursor into a string that returns Items one by one.
102///
103/// (This type isn't used directly, but is returned wrapped in a Peekable.)
104#[derive(Debug)]
105struct NetDocReaderBase<'a, K: Keyword> {
106    /// The string we're parsing.
107    s: &'a str,
108    /// Our position within the string.
109    off: usize,
110    /// Tells Rust it's okay that we are parameterizing on K.
111    _k: std::marker::PhantomData<K>,
112}
113
114impl<'a, K: Keyword> NetDocReaderBase<'a, K> {
115    /// Create a new NetDocReader to split a string into tokens.
116    fn new(s: &'a str) -> Result<Self> {
117        Ok(NetDocReaderBase {
118            s: validate_utf_8_rules(s)?,
119            off: 0,
120            _k: std::marker::PhantomData,
121        })
122    }
123    /// Return the current Pos within the string.
124    fn pos(&self, pos: usize) -> Pos {
125        Pos::from_offset(self.s, pos)
126    }
127    /// Skip forward by n bytes.
128    ///
129    /// (Note that standard caveats with byte-oriented processing of
130    /// UTF-8 strings apply.)
131    fn advance(&mut self, n: usize) -> Result<()> {
132        if n > self.remaining() {
133            return Err(
134                Error::from(internal!("tried to advance past end of document"))
135                    .at_pos(Pos::from_offset(self.s, self.off)),
136            );
137        }
138        self.off += n;
139        Ok(())
140    }
141    /// Return the remaining number of bytes in this reader.
142    fn remaining(&self) -> usize {
143        self.s.len() - self.off
144    }
145
146    /// Return true if the next characters in this reader are `s`
147    #[allow(clippy::string_slice)] // TODO
148    fn starts_with(&self, s: &str) -> bool {
149        self.s[self.off..].starts_with(s)
150    }
151    /// Try to extract a NL-terminated line from this reader.  Always
152    /// remove data if the reader is nonempty.
153    #[allow(clippy::string_slice)] // TODO
154    fn line(&mut self) -> Result<&'a str> {
155        let remainder = &self.s[self.off..];
156        if let Some(nl_pos) = remainder.find('\n') {
157            self.advance(nl_pos + 1)?;
158            let line = &remainder[..nl_pos];
159
160            // TODO: we should probably detect \r and do something about it.
161            // Just ignoring it isn't the right answer, though.
162            Ok(line)
163        } else {
164            self.advance(remainder.len())?; // drain everything.
165            Err(EK::TruncatedLine.at_pos(self.pos(self.s.len())))
166        }
167    }
168
169    /// Try to extract a line that begins with a keyword from this reader.
170    ///
171    /// Returns a (kwd, args) tuple on success.
172    #[allow(clippy::string_slice)] // TODO
173    fn kwdline(&mut self) -> Result<(&'a str, &'a str)> {
174        let pos = self.off;
175        let line = self.line()?;
176        if line.is_empty() {
177            return Err(EK::EmptyLine.at_pos(self.pos(pos)));
178        }
179        let (line, anno_ok) = if let Some(rem) = line.strip_prefix("opt ") {
180            (rem.trim_start_matches(is_sp), false)
181        } else {
182            (line, true)
183        };
184        let parts = line.split_once(is_sp);
185        let (kwd, args) = match parts {
186            Some((left, right)) => (left, right.trim_start_matches(is_sp)),
187            None => (line, &line[line.len()..]),
188        };
189        if !keyword_ok(kwd, anno_ok) {
190            return Err(EK::BadKeyword.at_pos(self.pos(pos)));
191        }
192        Ok((kwd, args))
193    }
194
195    /// Try to extract an Object beginning wrapped within BEGIN/END tags.
196    ///
197    /// Returns Ok(Some(Object(...))) on success if an object is
198    /// found, Ok(None) if no object is found, and Err only if a
199    /// corrupt object is found.
200    #[allow(clippy::string_slice)] // TODO
201    fn object(&mut self) -> Result<Option<Object<'a>>> {
202        use object::*;
203
204        let pos = self.off;
205        if !self.starts_with(BEGIN_STR) {
206            return Ok(None);
207        }
208        let line = self.line()?;
209        if !line.ends_with(TAG_END) {
210            return Err(EK::BadObjectBeginTag.at_pos(self.pos(pos)));
211        }
212        let tag = &line[BEGIN_STR.len()..(line.len() - TAG_END.len())];
213        if !tag_keywords_ok(tag) {
214            return Err(EK::BadObjectBeginTag.at_pos(self.pos(pos)));
215        }
216        let datapos = self.off;
217        let (endlinepos, endline) = loop {
218            let p = self.off;
219            let line = self.line()?;
220            if line.starts_with(END_STR) {
221                break (p, line);
222            }
223            // Exit if this line isn't plausible base64.  Otherwise,
224            // an unterminated base64 block could potentially
225            // "consume" all the rest of the string, which would stop
226            // us from recovering.
227            b64check(line).map_err(|e| e.within(self.s))?;
228        };
229        let data = &self.s[datapos..endlinepos];
230        if !endline.ends_with(TAG_END) {
231            return Err(EK::BadObjectEndTag.at_pos(self.pos(endlinepos)));
232        }
233        let endtag = &endline[END_STR.len()..(endline.len() - TAG_END.len())];
234        if endtag != tag {
235            return Err(EK::BadObjectMismatchedTag.at_pos(self.pos(endlinepos)));
236        }
237        Ok(Some(Object { tag, data, endline }))
238    }
239
240    /// Read the next Item from this NetDocReaderBase.
241    ///
242    /// If successful, returns Ok(Some(Item)), or Ok(None) if exhausted.
243    /// Returns Err on failure.
244    ///
245    /// Always consumes at least one line if possible; always ends on a
246    /// line boundary if one exists.
247    fn item(&mut self) -> Result<Option<Item<'a, K>>> {
248        if self.remaining() == 0 {
249            return Ok(None);
250        }
251        let (kwd_str, args) = self.kwdline()?;
252        let object = self.object()?;
253        let split_args = RefCell::new(None);
254        let kwd = K::from_str(kwd_str);
255        Ok(Some(Item {
256            kwd,
257            kwd_str,
258            args,
259            split_args,
260            object,
261        }))
262    }
263}
264
265/// Return true iff 's' is a valid keyword or annotation.
266///
267/// (Only allow annotations if `anno_ok` is true.`
268#[allow(clippy::string_slice)] // TODO
269fn keyword_ok(mut s: &str, anno_ok: bool) -> bool {
270    /// Helper: return true if this character can appear in keywords.
271    fn kwd_char_ok(c: char) -> bool {
272        matches!(c,'A'..='Z' | 'a'..='z' |'0'..='9' | '-')
273    }
274
275    if s.is_empty() {
276        return false;
277    }
278    if anno_ok && s.starts_with('@') {
279        s = &s[1..];
280    }
281    if s.starts_with('-') {
282        return false;
283    }
284    s.chars().all(kwd_char_ok)
285}
286
287/// Return true iff 's' is a valid keywords string for a BEGIN/END tag.
288pub(crate) fn tag_keywords_ok(s: &str) -> bool {
289    s.split(' ').all(|w| keyword_ok(w, false))
290}
291
292/// When used as an Iterator, returns a sequence of `Result<Item>`.
293impl<'a, K: Keyword> Iterator for NetDocReaderBase<'a, K> {
294    type Item = Result<Item<'a, K>>;
295    fn next(&mut self) -> Option<Self::Item> {
296        self.item().transpose()
297    }
298}
299
300/// Helper: as base64::decode(), but allows newlines in the middle of the
301/// encoded object.
302pub(crate) fn base64_decode_multiline(s: &str) -> std::result::Result<Vec<u8>, base64ct::Error> {
303    // base64 module hates whitespace.
304    let mut s = s.to_string();
305    s.retain(|ch| ch != '\n');
306    let v = Base64::decode_vec(&s)?;
307    Ok(v)
308}
309
310impl<'a, K: Keyword> Item<'a, K> {
311    /// Return the parsed keyword part of this item.
312    pub(crate) fn kwd(&self) -> K {
313        self.kwd
314    }
315    /// Return the keyword part of this item, as a string.
316    pub(crate) fn kwd_str(&self) -> &'a str {
317        self.kwd_str
318    }
319    /// Return true if the keyword for this item is in 'ks'.
320    pub(crate) fn has_kwd_in(&self, ks: &[K]) -> bool {
321        ks.contains(&self.kwd)
322    }
323    /// Return the arguments of this item, as a single string.
324    pub(crate) fn args_as_str(&self) -> &'a str {
325        self.args
326    }
327    /// Return the arguments of this item as a vector.
328    fn args_as_vec(&self) -> Ref<'_, Vec<&'a str>> {
329        // We're using an interior mutability pattern here to lazily
330        // construct the vector.
331        if self.split_args.borrow().is_none() {
332            self.split_args.replace(Some(self.args().collect()));
333        }
334        Ref::map(self.split_args.borrow(), |opt| match opt {
335            Some(v) => v,
336            None => panic!(),
337        })
338    }
339    /// Return an iterator over the arguments of this item.
340    pub(crate) fn args(&self) -> impl Iterator<Item = &'a str> + use<'a, K> {
341        self.args.split(is_sp).filter(|s| !s.is_empty())
342    }
343    /// Return the nth argument of this item, if there is one.
344    pub(crate) fn arg(&self, idx: usize) -> Option<&'a str> {
345        self.args_as_vec().get(idx).copied()
346    }
347    /// Return the nth argument of this item, or an error if it isn't there.
348    pub(crate) fn required_arg(&self, idx: usize) -> Result<&'a str> {
349        self.arg(idx)
350            .ok_or_else(|| EK::MissingArgument.at_pos(Pos::at(self.args)))
351    }
352    /// Try to parse the nth argument (if it exists) into some type
353    /// that supports FromStr.
354    ///
355    /// Returns Ok(None) if the argument doesn't exist.
356    pub(crate) fn parse_optional_arg<V: FromStr>(&self, idx: usize) -> Result<Option<V>>
357    where
358        Error: From<V::Err>,
359    {
360        match self.arg(idx) {
361            None => Ok(None),
362            Some(s) => match s.parse() {
363                Ok(r) => Ok(Some(r)),
364                Err(e) => {
365                    let e: Error = e.into();
366                    Err(e.or_at_pos(Pos::at(s)))
367                }
368            },
369        }
370    }
371    /// Try to parse the nth argument (if it exists) into some type
372    /// that supports FromStr.
373    ///
374    /// Return an error if the argument doesn't exist.
375    pub(crate) fn parse_arg<V: FromStr>(&self, idx: usize) -> Result<V>
376    where
377        Error: From<V::Err>,
378    {
379        match self.parse_optional_arg(idx) {
380            Ok(Some(v)) => Ok(v),
381            Ok(None) => Err(EK::MissingArgument.at_pos(self.arg_pos(idx))),
382            Err(e) => Err(e),
383        }
384    }
385    /// Return the number of arguments for this Item
386    pub(crate) fn n_args(&self) -> usize {
387        self.args().count()
388    }
389    /// Return true iff this Item has an associated object.
390    pub(crate) fn has_obj(&self) -> bool {
391        self.object.is_some()
392    }
393    /// Return the tag of this item's associated object, if it has one.
394    pub(crate) fn obj_tag(&self) -> Option<&'a str> {
395        self.object.map(|o| o.tag)
396    }
397    /// Try to decode the base64 contents of this Item's associated object.
398    ///
399    /// On success, return the object's tag and decoded contents.
400    pub(crate) fn obj_raw(&self) -> Result<Option<(&'a str, Vec<u8>)>> {
401        match self.object {
402            None => Ok(None),
403            Some(obj) => {
404                let decoded = base64_decode_multiline(obj.data)
405                    .map_err(|_| EK::BadObjectBase64.at_pos(Pos::at(obj.data)))?;
406                Ok(Some((obj.tag, decoded)))
407            }
408        }
409    }
410    /// Try to decode the base64 contents of this Item's associated object,
411    /// and make sure that its tag matches 'want_tag'.
412    pub(crate) fn obj(&self, want_tag: &str) -> Result<Vec<u8>> {
413        match self.obj_raw()? {
414            None => Err(EK::MissingObject
415                .with_msg(self.kwd.to_str())
416                .at_pos(self.end_pos())),
417            Some((tag, decoded)) => {
418                if tag != want_tag {
419                    Err(EK::WrongObject.at_pos(Pos::at(tag)))
420                } else {
421                    Ok(decoded)
422                }
423            }
424        }
425    }
426    /// Try to decode the base64 contents of this item's associated object
427    /// as a given type that implements FromBytes.
428    pub(crate) fn parse_obj<V: FromBytes>(&self, want_tag: &str) -> Result<V> {
429        let bytes = self.obj(want_tag)?;
430        // Unwrap may be safe because above `.obj()` should return an Error if
431        // wanted tag was not present
432        #[allow(clippy::unwrap_used)]
433        let p = Pos::at(self.object.unwrap().data);
434        V::from_vec(bytes, p).map_err(|e| e.at_pos(p))
435    }
436    /// Return the position of this item.
437    ///
438    /// This position won't be useful unless it is later contextualized
439    /// with the containing string.
440    pub(crate) fn pos(&self) -> Pos {
441        Pos::at(self.kwd_str)
442    }
443    /// Return the position of this Item in a string.
444    ///
445    /// Returns None if this item doesn't actually belong to the string.
446    pub(crate) fn offset_in(&self, s: &str) -> Option<usize> {
447        crate::util::str::str_offset(s, self.kwd_str)
448    }
449    /// Return the position of the n'th argument of this item.
450    ///
451    /// If this item does not have a n'th argument, return the
452    /// position of the end of the final argument.
453    pub(crate) fn arg_pos(&self, n: usize) -> Pos {
454        let args = self.args_as_vec();
455        if n < args.len() {
456            Pos::at(args[n])
457        } else {
458            self.last_arg_end_pos()
459        }
460    }
461    /// Return the position at the end of the last argument.  (This will
462    /// point to a newline.)
463    fn last_arg_end_pos(&self) -> Pos {
464        Pos::at_end_of(self.args)
465    }
466    /// Return the position of the end of this object. (This will point to a
467    /// newline.)
468    pub(crate) fn end_pos(&self) -> Pos {
469        match self.object {
470            Some(o) => Pos::at_end_of(o.endline),
471            None => self.last_arg_end_pos(),
472        }
473    }
474    /// If this item occurs within s, return the byte offset
475    /// immediately after the end of this item.
476    pub(crate) fn offset_after(&self, s: &str) -> Option<usize> {
477        self.end_pos().offset_within(s).map(|nl_pos| nl_pos + 1)
478    }
479
480    /// Return the text of this item, if it originated within `str`,
481    /// from the start of its keyword up to and including its final newline.
482    #[allow(dead_code)] // unused when hsdesc not enabled.
483    pub(crate) fn text_within<'b>(&self, s: &'b str) -> Option<&'b str> {
484        let start = self.pos().offset_within(s)?;
485        let end = self.end_pos().offset_within(s)?;
486        s.get(start..=end)
487    }
488}
489
490/// Represents an Item that might not be present, whose arguments we
491/// want to inspect.  If the Item is there, this acts like a proxy to the
492/// item; otherwise, it treats the item as having no arguments.
493pub(crate) struct MaybeItem<'a, 'b, K: Keyword>(Option<&'a Item<'b, K>>);
494
495// All methods here are as for Item.
496impl<'a, 'b, K: Keyword> MaybeItem<'a, 'b, K> {
497    /// Return the position of this item, if it has one.
498    pub(crate) fn pos(&self) -> Pos {
499        match self.0 {
500            Some(item) => item.pos(),
501            None => Pos::None,
502        }
503    }
504    /// Construct a MaybeItem from an Option reference to an item.
505    pub(crate) fn from_option(opt: Option<&'a Item<'b, K>>) -> Self {
506        MaybeItem(opt)
507    }
508
509    /// If this item is present, parse its argument at position `idx`.
510    /// Treat the absence or malformedness of the argument as an error,
511    /// but treat the absence of this item as acceptable.
512    pub(crate) fn parse_arg<V: FromStr>(&self, idx: usize) -> Result<Option<V>>
513    where
514        Error: From<V::Err>,
515    {
516        match self.0 {
517            Some(item) => match item.parse_arg(idx) {
518                Ok(v) => Ok(Some(v)),
519                Err(e) => Err(e.or_at_pos(self.pos())),
520            },
521            None => Ok(None),
522        }
523    }
524    /// If this item is present, return its arguments as a single string.
525    pub(crate) fn args_as_str(&self) -> Option<&str> {
526        self.0.map(|item| item.args_as_str())
527    }
528    /// If this item is present, parse all of its arguments as a
529    /// single string.
530    pub(crate) fn parse_args_as_str<V: FromStr>(&self) -> Result<Option<V>>
531    where
532        Error: From<V::Err>,
533    {
534        match self.0 {
535            Some(item) => match item.args_as_str().parse::<V>() {
536                Ok(v) => Ok(Some(v)),
537                Err(e) => {
538                    let e: Error = e.into();
539                    Err(e.or_at_pos(self.pos()))
540                }
541            },
542            None => Ok(None),
543        }
544    }
545}
546
547/// Extension trait for `Result<Item>` -- makes it convenient to implement
548/// PauseAt predicates
549pub(crate) trait ItemResult<K: Keyword> {
550    /// Return true if this is an ok result with an annotation.
551    fn is_ok_with_annotation(&self) -> bool;
552    /// Return true if this is an ok result with a non-annotation.
553    fn is_ok_with_non_annotation(&self) -> bool;
554    /// Return true if this is an ok result with the keyword 'k'
555    fn is_ok_with_kwd(&self, k: K) -> bool {
556        self.is_ok_with_kwd_in(&[k])
557    }
558    /// Return true if this is an ok result with a keyword in the slice 'ks'
559    fn is_ok_with_kwd_in(&self, ks: &[K]) -> bool;
560    /// Return true if this is an ok result with a keyword not in the slice 'ks'
561    fn is_ok_with_kwd_not_in(&self, ks: &[K]) -> bool;
562    /// Return true if this is an empty-line error.
563    fn is_empty_line(&self) -> bool;
564}
565
566impl<'a, K: Keyword> ItemResult<K> for Result<Item<'a, K>> {
567    fn is_ok_with_annotation(&self) -> bool {
568        match self {
569            Ok(item) => item.kwd().is_annotation(),
570            Err(_) => false,
571        }
572    }
573    fn is_ok_with_non_annotation(&self) -> bool {
574        match self {
575            Ok(item) => !item.kwd().is_annotation(),
576            Err(_) => false,
577        }
578    }
579    fn is_ok_with_kwd_in(&self, ks: &[K]) -> bool {
580        match self {
581            Ok(item) => item.has_kwd_in(ks),
582            Err(_) => false,
583        }
584    }
585    fn is_ok_with_kwd_not_in(&self, ks: &[K]) -> bool {
586        match self {
587            Ok(item) => !item.has_kwd_in(ks),
588            Err(_) => false,
589        }
590    }
591    fn is_empty_line(&self) -> bool {
592        matches!(
593            self,
594            Err(e) if e.netdoc_error_kind() == crate::err::NetdocErrorKind::EmptyLine
595        )
596    }
597}
598
599/// A peekable cursor into a string that returns Items one by one.
600///
601/// This is an [`Iterator`], yielding [`Item`]s.
602#[derive(Debug)]
603pub(crate) struct NetDocReader<'a, K: Keyword> {
604    // TODO: I wish there were some way around having this string
605    // reference, since we already need one inside NetDocReaderBase.
606    /// The underlying string being parsed.
607    s: &'a str,
608    /// A stream of tokens being parsed by this NetDocReader.
609    tokens: Peekable<NetDocReaderBase<'a, K>>,
610}
611
612impl<'a, K: Keyword> NetDocReader<'a, K> {
613    /// Construct a new NetDocReader to read tokens from `s`.
614    pub(crate) fn new(s: &'a str) -> Result<Self> {
615        Ok(NetDocReader {
616            s,
617            tokens: NetDocReaderBase::new(s)?.peekable(),
618        })
619    }
620    /// Return a reference to the string used for this NetDocReader.
621    pub(crate) fn str(&self) -> &'a str {
622        self.s
623    }
624    /// Return a wrapper around the peekable iterator in this
625    /// NetDocReader that reads tokens until it reaches an element where
626    /// 'f' is true.
627    pub(crate) fn pause_at<'f, 'r, F>(
628        &mut self,
629        mut f: F,
630    ) -> itertools::PeekingTakeWhile<
631        '_,
632        Self,
633        impl FnMut(&Result<Item<'a, K>>) -> bool + 'f + use<'a, 'f, F, K>,
634    >
635    where
636        'f: 'r,
637        F: FnMut(&Result<Item<'a, K>>) -> bool + 'f,
638        K: 'f,
639    {
640        self.peeking_take_while(move |i| !f(i))
641    }
642
643    /// Return true if there are no more items in this NetDocReader.
644    // The implementation sadly needs to mutate the inner state, even if it's not *semantically*
645    // mutated..  We don't want inner mutability just to placate clippy for an internal API.
646    #[allow(clippy::wrong_self_convention)]
647    #[allow(dead_code)] // TODO perhaps we should remove this ?
648    pub(crate) fn is_exhausted(&mut self) -> bool {
649        self.peek().is_none()
650    }
651
652    /// Give an error if there are remaining tokens in this NetDocReader.
653    pub(crate) fn should_be_exhausted(&mut self) -> Result<()> {
654        match self.peek() {
655            None => Ok(()),
656            Some(Ok(t)) => Err(EK::UnexpectedToken
657                .with_msg(t.kwd().to_str())
658                .at_pos(t.pos())),
659            Some(Err(e)) => Err(e.clone()),
660        }
661    }
662
663    /// Give an error if there are remaining tokens in this NetDocReader.
664    ///
665    /// Like [`should_be_exhausted`](Self::should_be_exhausted),
666    /// but permit empty lines at the end of the document.
667    pub(crate) fn should_be_exhausted_but_for_empty_lines(&mut self) -> Result<()> {
668        use crate::err::NetdocErrorKind as K;
669        while let Some(Err(e)) = self.peek() {
670            if e.netdoc_error_kind() == K::EmptyLine {
671                let _ignore = self.next();
672            } else {
673                break;
674            }
675        }
676        self.should_be_exhausted()
677    }
678
679    /// Return the position from which the underlying reader is about to take
680    /// the next token.  Use to make sure that the reader is progressing.
681    pub(crate) fn pos(&mut self) -> Pos {
682        match self.tokens.peek() {
683            Some(Ok(tok)) => tok.pos(),
684            Some(Err(e)) => e.pos(),
685            None => Pos::at_end_of(self.s),
686        }
687    }
688}
689
690impl<'a, K: Keyword> Iterator for NetDocReader<'a, K> {
691    type Item = Result<Item<'a, K>>;
692    fn next(&mut self) -> Option<Self::Item> {
693        self.tokens.next()
694    }
695}
696
697impl<'a, K: Keyword> PeekableIterator for NetDocReader<'a, K> {
698    fn peek(&mut self) -> Option<&Self::Item> {
699        self.tokens.peek()
700    }
701}
702
703impl<'a, K: Keyword> itertools::PeekingNext for NetDocReader<'a, K> {
704    fn peeking_next<F>(&mut self, f: F) -> Option<Self::Item>
705    where
706        F: FnOnce(&Self::Item) -> bool,
707    {
708        if f(self.peek()?) { self.next() } else { None }
709    }
710}
711
712/// Check additional UTF-8 rules that the netdoc metaformat imposes on
713/// our documents.
714//
715// NOTE: We might decide in the future to loosen our rules here
716// for parsers that handle concatenated documents:
717// we might want to reject only those documents that contain NULs.
718// But with luck that will never be necessary.
719fn validate_utf_8_rules(s: &str) -> Result<&str> {
720    // No BOM, or mangled BOM, is allowed.
721    let first_char = s.chars().next();
722    if [Some('\u{feff}'), Some('\u{fffe}')].contains(&first_char) {
723        return Err(EK::BomMarkerFound.at_pos(Pos::at(s)));
724    }
725    // No NUL bytes are allowed.
726    if let Some(nul_pos) = memchr::memchr(0, s.as_bytes()) {
727        return Err(EK::NulFound.at_pos(Pos::from_byte(nul_pos)));
728    }
729    Ok(s)
730}
731
732#[cfg(test)]
733mod test {
734    // @@ begin test lint list maintained by maint/add_warning @@
735    #![allow(clippy::bool_assert_comparison)]
736    #![allow(clippy::clone_on_copy)]
737    #![allow(clippy::dbg_macro)]
738    #![allow(clippy::mixed_attributes_style)]
739    #![allow(clippy::print_stderr)]
740    #![allow(clippy::print_stdout)]
741    #![allow(clippy::single_char_pattern)]
742    #![allow(clippy::unwrap_used)]
743    #![allow(clippy::unchecked_time_subtraction)]
744    #![allow(clippy::useless_vec)]
745    #![allow(clippy::needless_pass_by_value)]
746    #![allow(clippy::string_slice)] // See arti#2571
747    //! <!-- @@ end test lint list maintained by maint/add_warning @@ -->
748    use super::*;
749    use crate::parse::macros::test::Fruit;
750    use crate::{NetdocErrorKind as EK, Pos, Result};
751
752    #[test]
753    fn read_simple() {
754        use Fruit::*;
755
756        let s = "\
757@tasty very much so
758opt apple 77
759banana 60
760cherry 6
761-----BEGIN CHERRY SYNOPSIS-----
7628J+NkvCfjZLwn42S8J+NkvCfjZLwn42S
763-----END CHERRY SYNOPSIS-----
764plum hello there
765orange    77
766opt    lemon    tabs    and    spaces
767";
768        let mut r: NetDocReader<'_, Fruit> = NetDocReader::new(s).unwrap();
769
770        assert_eq!(r.str(), s);
771        assert!(r.should_be_exhausted().is_err()); // it's not exhausted.
772
773        let toks: Result<Vec<_>> = r.by_ref().collect();
774        assert!(r.should_be_exhausted().is_ok());
775
776        let toks = toks.unwrap();
777        assert_eq!(toks.len(), 7);
778        assert_eq!(toks[0].kwd(), ANN_TASTY);
779        assert_eq!(toks[0].n_args(), 3);
780        assert_eq!(toks[0].args_as_str(), "very much so");
781        assert_eq!(toks[0].arg(1), Some("much"));
782        {
783            let a: Vec<_> = toks[0].args().collect();
784            assert_eq!(a, vec!["very", "much", "so"]);
785        }
786        assert!(toks[0].parse_arg::<usize>(0).is_err());
787        assert!(toks[0].parse_arg::<usize>(10).is_err());
788        assert!(!toks[0].has_obj());
789        assert_eq!(toks[0].obj_tag(), None);
790
791        assert_eq!(toks[2].pos().within(s), Pos::from_line(3, 1));
792        assert_eq!(toks[2].arg_pos(0).within(s), Pos::from_line(3, 8));
793        assert_eq!(toks[2].last_arg_end_pos().within(s), Pos::from_line(3, 10));
794        assert_eq!(toks[2].end_pos().within(s), Pos::from_line(3, 10));
795
796        assert_eq!(toks[3].kwd(), STONEFRUIT);
797        assert_eq!(toks[3].kwd_str(), "cherry"); // not cherry/plum!
798        assert_eq!(toks[3].n_args(), 1);
799        assert_eq!(toks[3].required_arg(0), Ok("6"));
800        assert_eq!(toks[3].parse_arg::<usize>(0), Ok(6));
801        assert_eq!(toks[3].parse_optional_arg::<usize>(0), Ok(Some(6)));
802        assert_eq!(toks[3].parse_optional_arg::<usize>(3), Ok(None));
803        assert!(toks[3].has_obj());
804        assert_eq!(toks[3].obj_tag(), Some("CHERRY SYNOPSIS"));
805        assert_eq!(
806            &toks[3].obj("CHERRY SYNOPSIS").unwrap()[..],
807            "🍒🍒🍒🍒🍒🍒".as_bytes()
808        );
809        assert!(toks[3].obj("PLUOT SYNOPSIS").is_err());
810        // this "end-pos" value is questionable!
811        assert_eq!(toks[3].end_pos().within(s), Pos::from_line(7, 30));
812
813        assert_eq!(toks[5].kwd(), ORANGE);
814        assert_eq!(toks[5].n_args(), 1);
815        assert_eq!(toks[5].args_as_str(), "77");
816
817        assert_eq!(toks[6].kwd(), LEMON);
818        assert_eq!(toks[6].n_args(), 3);
819        assert_eq!(toks[6].args_as_str(), "tabs    and    spaces");
820        assert_eq!(toks[6].required_arg(0), Ok("tabs"));
821        assert_eq!(toks[6].required_arg(1), Ok("and"));
822        assert_eq!(toks[6].required_arg(2), Ok("spaces"));
823    }
824
825    #[test]
826    fn test_badtoks() {
827        use Fruit::*;
828
829        let s = "\
830-foobar 9090
831apple 3.14159
832$hello
833unrecognized 127.0.0.1 foo
834plum
835-----BEGIN WHATEVER-----
8368J+NkvCfjZLwn42S8J+NkvCfjZLwn42S
837-----END SOMETHING ELSE-----
838orange
839orange
840-----BEGIN WHATEVER-----
841not! base64!
842-----END WHATEVER-----
843guava paste
844opt @annotation
845orange
846-----BEGIN LOBSTER
8478J+NkvCfjZLwn42S8J+NkvCfjZLwn42S
848-----END SOMETHING ELSE-----
849orange
850-----BEGIN !!!!!!-----
8518J+NkvCfjZLwn42S8J+NkvCfjZLwn42S
852-----END !!!!!!-----
853cherry
854-----BEGIN CHERRY SYNOPSIS-----
8558J+NkvCfjZLwn42S8J+NkvCfjZLwn42S
856-----END CHERRY SYNOPSIS
857
858truncated line";
859
860        let r: NetDocReader<'_, Fruit> = NetDocReader::new(s).unwrap();
861        let toks: Vec<_> = r.collect();
862
863        assert!(toks[0].is_err());
864        assert_eq!(
865            toks[0].as_ref().err().unwrap(),
866            &EK::BadKeyword.at_pos(Pos::from_line(1, 1))
867        );
868
869        assert!(toks[1].is_ok());
870        assert!(toks[1].is_ok_with_non_annotation());
871        assert!(!toks[1].is_ok_with_annotation());
872        assert!(toks[1].is_ok_with_kwd_in(&[APPLE, ORANGE]));
873        assert!(toks[1].is_ok_with_kwd_not_in(&[ORANGE, UNRECOGNIZED]));
874        let t = toks[1].as_ref().unwrap();
875        assert_eq!(t.kwd(), APPLE);
876        assert_eq!(t.arg(0), Some("3.14159"));
877
878        assert!(toks[2].is_err());
879        assert!(!toks[2].is_ok_with_non_annotation());
880        assert!(!toks[2].is_ok_with_annotation());
881        assert!(!toks[2].is_ok_with_kwd_in(&[APPLE, ORANGE]));
882        assert!(!toks[2].is_ok_with_kwd_not_in(&[ORANGE, UNRECOGNIZED]));
883        assert_eq!(
884            toks[2].as_ref().err().unwrap(),
885            &EK::BadKeyword.at_pos(Pos::from_line(3, 1))
886        );
887
888        assert!(toks[3].is_ok());
889        let t = toks[3].as_ref().unwrap();
890        assert_eq!(t.kwd(), UNRECOGNIZED);
891        assert_eq!(t.arg(1), Some("foo"));
892
893        assert!(toks[4].is_err());
894        assert_eq!(
895            toks[4].as_ref().err().unwrap(),
896            &EK::BadObjectMismatchedTag.at_pos(Pos::from_line(8, 1))
897        );
898
899        assert!(toks[5].is_ok());
900        let t = toks[5].as_ref().unwrap();
901        assert_eq!(t.kwd(), ORANGE);
902        assert_eq!(t.args_as_str(), "");
903
904        // This blob counts as two errors: a bad base64 blob, and
905        // then an end line.
906        assert!(toks[6].is_err());
907        assert_eq!(
908            toks[6].as_ref().err().unwrap(),
909            &EK::BadObjectBase64.at_pos(Pos::from_line(12, 1))
910        );
911
912        assert!(toks[7].is_err());
913        assert_eq!(
914            toks[7].as_ref().err().unwrap(),
915            &EK::BadKeyword.at_pos(Pos::from_line(13, 1))
916        );
917
918        assert!(toks[8].is_ok());
919        let t = toks[8].as_ref().unwrap();
920        assert_eq!(t.kwd(), GUAVA);
921
922        // this is an error because you can't use opt with annotations.
923        assert!(toks[9].is_err());
924        assert_eq!(
925            toks[9].as_ref().err().unwrap(),
926            &EK::BadKeyword.at_pos(Pos::from_line(15, 1))
927        );
928
929        // this looks like a few errors.
930        assert!(toks[10].is_err());
931        assert_eq!(
932            toks[10].as_ref().err().unwrap(),
933            &EK::BadObjectBeginTag.at_pos(Pos::from_line(17, 1))
934        );
935        assert!(toks[11].is_err());
936        assert_eq!(
937            toks[11].as_ref().err().unwrap(),
938            &EK::BadKeyword.at_pos(Pos::from_line(18, 1))
939        );
940        assert!(toks[12].is_err());
941        assert_eq!(
942            toks[12].as_ref().err().unwrap(),
943            &EK::BadKeyword.at_pos(Pos::from_line(19, 1))
944        );
945
946        // so does this.
947        assert!(toks[13].is_err());
948        assert_eq!(
949            toks[13].as_ref().err().unwrap(),
950            &EK::BadObjectBeginTag.at_pos(Pos::from_line(21, 1))
951        );
952        assert!(toks[14].is_err());
953        assert_eq!(
954            toks[14].as_ref().err().unwrap(),
955            &EK::BadKeyword.at_pos(Pos::from_line(22, 1))
956        );
957        assert!(toks[15].is_err());
958        assert_eq!(
959            toks[15].as_ref().err().unwrap(),
960            &EK::BadKeyword.at_pos(Pos::from_line(23, 1))
961        );
962
963        // not this.
964        assert!(toks[16].is_err());
965        assert_eq!(
966            toks[16].as_ref().err().unwrap(),
967            &EK::BadObjectEndTag.at_pos(Pos::from_line(27, 1))
968        );
969
970        assert!(toks[17].is_err());
971        assert_eq!(
972            toks[17].as_ref().err().unwrap(),
973            &EK::EmptyLine.at_pos(Pos::from_line(28, 1))
974        );
975
976        assert!(toks[18].is_err());
977        assert_eq!(
978            toks[18].as_ref().err().unwrap(),
979            &EK::TruncatedLine.at_pos(Pos::from_line(29, 15))
980        );
981    }
982
983    #[test]
984    fn test_leading_space_forbidden() {
985        // We need to make sure that items with a leading space aren't accepted:
986        // the spec forbids it, and it can provide a vector for inflating the size
987        // of downloaded hsdescs (see prop360).
988
989        // Try a simple item with a space at the front.
990        let s = "    guava space\n";
991        let r: NetDocReader<'_, Fruit> = NetDocReader::new(s).unwrap();
992        let toks: Vec<_> = r.collect();
993
994        // No space allowed at the start of a line.
995        assert_eq!(
996            toks[0].as_ref().err().unwrap(),
997            &EK::BadKeyword.at_pos(Pos::from_line(1, 1))
998        );
999
1000        // Try an item with an object, inserting space at the start of each ine in turn.
1001        let s = "cherry
1002-----BEGIN WHATEVER-----
10038J+NkvCfjZLwn42S8J+NkvCfjZLwn42S
1004-----END WHATEVER-----
1005";
1006
1007        let orig_lines = s
1008            .split_terminator('\n')
1009            .map(str::to_string)
1010            .collect::<Vec<_>>();
1011        assert_eq!(orig_lines.len(), 4);
1012        let expected_kinds = [
1013            EK::BadKeyword,
1014            EK::BadKeyword,
1015            EK::BadObjectBase64,
1016            EK::BadObjectBase64,
1017        ];
1018        for pos in 0..orig_lines.len() {
1019            let mut lines = orig_lines.clone();
1020            lines[pos] = format!(" {}", lines[pos]);
1021            let joined = format!("{}\n", lines.join("\n"));
1022
1023            let r: NetDocReader<'_, Fruit> = NetDocReader::new(&joined).unwrap();
1024            let toks: Result<Vec<_>> = r.collect();
1025            assert_eq!(toks.unwrap_err().netdoc_error_kind(), expected_kinds[pos]);
1026        }
1027    }
1028
1029    #[test]
1030    fn test_validate_strings() {
1031        use validate_utf_8_rules as v;
1032        assert_eq!(v(""), Ok(""));
1033        assert_eq!(v("hello world"), Ok("hello world"));
1034        // We don't have to test a lot more valid cases, since this function is called before
1035        // parsing any string.
1036
1037        for s in ["\u{feff}", "\u{feff}hello world", "\u{fffe}hello world"] {
1038            let e = v(s).unwrap_err();
1039            assert_eq!(e.netdoc_error_kind(), EK::BomMarkerFound);
1040            assert_eq!(e.pos().offset_within(s), Some(0));
1041        }
1042
1043        for s in [
1044            "\0hello world",
1045            "\0",
1046            "\0\0\0",
1047            "hello\0world",
1048            "hello world\0",
1049        ] {
1050            let e = v(s).unwrap_err();
1051            assert_eq!(e.netdoc_error_kind(), EK::NulFound);
1052            let nul_pos = e.pos().offset_within(s).unwrap();
1053            assert_eq!(s.as_bytes()[nul_pos], 0);
1054        }
1055    }
1056
1057    fn single_fruit(s: &str) -> Item<'_, Fruit> {
1058        NetDocReader::<Fruit>::new(s)
1059            .unwrap()
1060            .next()
1061            .unwrap()
1062            .unwrap()
1063    }
1064
1065    #[test]
1066    fn end_of_item() {
1067        let s = "guava friends 123   \n";
1068        let item = single_fruit(s);
1069        assert_eq!(
1070            item.end_pos().within(s),
1071            Pos::from_byte(s.find('\n').unwrap()).within(s)
1072        );
1073
1074        let s = "cherry
1075-----BEGIN WHATEVER-----
10768J+NkvCfjZLwn42S8J+NkvCfjZLwn42S
1077-----END WHATEVER-----\n";
1078        let item = single_fruit(s);
1079        dbg!(&item);
1080        assert_eq!(
1081            item.end_pos().within(s),
1082            Pos::from_byte(s.rfind('\n').unwrap()).within(s)
1083        );
1084    }
1085}