tor_netdoc/parse2/lex.rs
1//! Lexing of netdoc elements
2
3use super::*;
4
5/// Linear whitespace as defined by torspec
6// Only pub via internal_prelude, for benefit of macros
7pub const WS: &[char] = &[' ', '\t'];
8
9define_derive_deftly! {
10 /// Define `parse_options` accessor
11 ///
12 /// The driver must have a lifetime named `'s`, which is suitable for the returned
13 /// `&'s ParseOptions`.
14 ///
15 /// # Top-level attributes:
16 ///
17 /// * **`#[deftly(parse_options(field = ".field.field"))]`**, default `.options`
18 ParseOptions beta_deftly, expect items:
19
20 $impl {
21 /// Examine the parsing options
22 pub fn parse_options(&self) -> &'s ParseOptions {
23 &self
24 ${tmeta(parse_options(field))
25 as token_stream,
26 default { .options }}
27 }
28 }
29}
30
31/// Top-level reader: Netdoc text interpreted as a stream of items
32#[derive(Debug, Clone, Deftly)]
33#[derive_deftly(ParseOptions)]
34pub struct ItemStream<'s> {
35 /// The whole input document.
36 whole_input: &'s str,
37 /// Remaining document, as a stream of lines
38 lines: Lines<'s>,
39 /// If we have peeked ahead, what we discovered
40 peeked: PeekState<'s>,
41 /// Parsing options.
42 options: &'s ParseOptions,
43}
44
45/// Whether an `ItemStream` has peeked ahead, and if so what it discovered
46#[derive(Debug, Clone)]
47enum PeekState<'s> {
48 /// We've peeked a line
49 Some(ItemStreamPeeked<'s>),
50 /// We've not peeked, or peeking gave `None`
51 None {
52 /// Line number of the last item we yielded.
53 ///
54 /// `0` at the start.
55 yielded_item_lno: usize,
56 },
57}
58
59/// If an `ItemStream` has peeked ahead, what it discovered
60#[derive(Debug, Clone)]
61struct ItemStreamPeeked<'s> {
62 /// The next keyword
63 keyword: KeywordRef<'s>,
64 /// Token proving that we
65 line: lines::Peeked,
66 /// Length of the suffix of the line that is the arguments rather than the keyword
67 ///
68 /// Does not include the first whitespace, that terminated the keyword.
69 args_len: usize,
70}
71
72/// An Item that has been lexed but not parsed
73#[derive(Debug, Clone, amplify::Getters, Deftly)]
74#[derive_deftly(ParseOptions)]
75#[deftly(parse_options(field = ".args.options"))]
76pub struct UnparsedItem<'s> {
77 /// The item's Keyword
78 #[getter(as_copy)]
79 keyword: KeywordRef<'s>,
80 /// The Item's Arguments
81 #[getter(skip)]
82 args: ArgumentStream<'s>,
83 /// The Item's Object, if there was one
84 #[getter(as_clone)]
85 object: Option<UnparsedObject<'s>>,
86}
87
88/// Reader for arguments on an Item
89///
90/// Represents the (remaining) arguments.
91#[derive(Debug, Clone, Deftly)]
92#[derive_deftly(ParseOptions)]
93pub struct ArgumentStream<'s> {
94 /// The remaining unparsed arguments
95 ///
96 /// Can start with WS, which is usually trimmed
97 rest: &'s str,
98
99 /// Original line length
100 ///
101 /// Used for reporting column of argument errors.
102 whole_line_len: usize,
103
104 /// Remaining length *before* we last yielded.
105 previous_rest_len: usize,
106
107 /// Parsing options.
108 options: &'s ParseOptions,
109}
110
111/// An Object that has been lexed but not parsed
112#[derive(Debug, Clone, amplify::Getters, Deftly)]
113#[derive_deftly(ParseOptions)]
114pub struct UnparsedObject<'s> {
115 /// The Label
116 #[getter(as_copy)]
117 label: &'s str,
118
119 /// The portion of the input document which is base64 data (and newlines)
120 #[getter(skip)]
121 data_b64: &'s str,
122
123 /// Parsing options.
124 options: &'s ParseOptions,
125}
126
127impl<'s> ItemStream<'s> {
128 /// Start reading a network document as a series of Items
129 pub fn new(input: &'s ParseInput<'s>) -> Self {
130 ItemStream {
131 whole_input: input.input,
132 lines: Lines::new(input.input),
133 peeked: PeekState::None {
134 yielded_item_lno: 0,
135 },
136 options: &input.options,
137 }
138 }
139
140 /// Line number for reporting an error we have just discovered
141 ///
142 /// If we have recent peeked, we report the line number of the peeked keyword line.
143 ///
144 /// Otherwise, we report the line number of the most-recently yielded item.
145 pub fn lno_for_error(&self) -> usize {
146 match self.peeked {
147 PeekState::Some { .. } => {
148 // The error was presumably caused by whatever was seen in the peek.
149 // That's the current line number.
150 self.lines.peek_lno()
151 }
152 PeekState::None { yielded_item_lno } => {
153 // The error was presumably caused by the results of next_item().
154 yielded_item_lno
155 }
156 }
157 }
158
159 /// Core of peeking. Tries to make `.peeked` be `Some`.
160 fn peek_internal<'i>(&'i mut self) -> Result<(), EP> {
161 if matches!(self.peeked, PeekState::None { .. }) {
162 let Some(peeked) = self.lines.peek() else {
163 return Ok(());
164 };
165
166 let peeked_line = self.lines.peeked_line(&peeked);
167
168 let (keyword, args) = peeked_line.split_once(WS).unwrap_or((peeked_line, ""));
169 let keyword = KeywordRef::new(keyword)?;
170
171 self.peeked = PeekState::Some(ItemStreamPeeked {
172 keyword,
173 line: peeked,
174 args_len: args.len(),
175 });
176 }
177
178 Ok(())
179 }
180
181 /// Peek the next keyword
182 pub fn peek_keyword(&mut self) -> Result<Option<KeywordRef<'s>>, EP> {
183 self.peek_internal()?;
184 let PeekState::Some(peeked) = &self.peeked else {
185 return Ok(None);
186 };
187 Ok(Some(peeked.keyword))
188 }
189
190 /// Obtain the body so far, suitable for hashing for an Orderly signature
191 #[allow(clippy::string_slice)] // TODO
192 pub fn body_sofar_for_signature(&self) -> SignedDocumentBody<'s> {
193 let body = &self.whole_input[0..self.byte_position()];
194 SignedDocumentBody { body }
195 }
196
197 /// Byte position, pointing to the start of the next item to yield
198 ///
199 /// Offset in bytes from the start of the original input string
200 /// to the "current" position,
201 /// ie to just after the item we yielded and just before the next item (or EOF).
202 pub fn byte_position(&self) -> usize {
203 self.whole_input.len() - self.lines.remaining().len()
204 }
205
206 /// Access for the entire input string
207 ///
208 /// The original `input: &str` argument to [`ParseInput::new`].
209 ///
210 /// Includes both yielded and unyielded items.
211 pub fn whole_input(&self) -> &'s str {
212 self.whole_input
213 }
214
215 /// Access the inner lines reader, mutably
216 ///
217 /// For special-purpose parsing situations (including unpleasant hacks).
218 /// For example, this can be used to skip `@`-annotations in C Tor document files.
219 ///
220 /// Pre- and post-condition: the `Lines` points at the next item to read.
221 pub fn with_inner_lines_mut<F, R>(&mut self, call: F) -> R
222 where
223 F: FnOnce(&mut Lines<'s>) -> R + std::panic::UnwindSafe,
224 {
225 use std::panic::{AssertUnwindSafe, catch_unwind, resume_unwind};
226
227 // The peeked state might be invalidated by `call`, so we must clear it.
228 // Let's take care to do this even if `call` panics.
229
230 let r = catch_unwind(AssertUnwindSafe(|| call(&mut self.lines)));
231 self.peeked = PeekState::None {
232 yielded_item_lno: self.lines.peek_lno(),
233 };
234 r.unwrap_or_else(|e| resume_unwind(e))
235 }
236
237 /// Parse a (sub-)document with its own signatures
238 ///
239 /// Used (mostly) by the
240 /// [`NetdocParseableUnverified`](derive_deftly_template_NetdocParseableUnverified)
241 /// derive macro.
242 ///
243 /// Generic parameters:
244 ///
245 /// * **`B`**: the body type: the type to which `NetdocParseableUnverified` is applied.
246 /// * **`S`**: the signatures section type.
247 /// * **`O`**: the `FooUnverified` type, which embodies the parsed body and signatures.
248 #[allow(clippy::string_slice)] // TODO
249 pub fn parse_signed<
250 B: HasUnverifiedParsedBody,
251 S: NetdocParseableSignatures,
252 O: NetdocParseableUnverified<Body = B, Signatures = S>,
253 >(
254 &mut self,
255 outer_stop: stop_at!(),
256 ) -> Result<O, EP> {
257 let mut input = ItemStream {
258 whole_input: &self.whole_input[self.whole_input.len() - self.lines.remaining().len()..],
259 ..self.clone()
260 };
261 let r = (|| {
262 let inner_always_stop = outer_stop | StopAt::doc_intro::<B::UnverifiedParsedBody>();
263 let body = B::UnverifiedParsedBody::from_items(
264 &mut input,
265 inner_always_stop | StopAt(S::is_item_keyword),
266 )?;
267 let signed_doc_body = input.body_sofar_for_signature();
268 let unsigned_body_len = signed_doc_body.body().len();
269 let mut hashes = S::HashesAccu::default();
270 let sigs = S::from_items(&mut input, signed_doc_body, &mut hashes, inner_always_stop)?;
271 let sigs = SignaturesData {
272 sigs,
273 unsigned_body_len,
274 hashes,
275 };
276 // SECURITY
277 // We unwrap the UnverifiedParsedBody and immediately wrap it up again
278 // in FooUnverified, passing on the obligation to verify the signatures,
279 // and still enforcing that with a newtype.
280 let signed = O::from_parts(B::unverified_into_inner_unchecked(body), sigs);
281 Ok(signed)
282 })(); // don't exit here
283
284 *self = ItemStream {
285 whole_input: self.whole_input,
286 ..input
287 };
288
289 r
290 }
291
292 /// Obtain the inputs that would be needed to hash any (even Disorderly) signature
293 ///
294 /// These are the hash inputs which would be needed for the next item,
295 /// assuming it's a signature keyword.
296 pub fn peek_signature_hash_inputs(
297 &mut self,
298 body: SignedDocumentBody<'s>,
299 ) -> Result<Option<SignatureHashInputs<'s>>, EP> {
300 self.peek_internal()?;
301 let PeekState::Some(peeked) = &self.peeked else {
302 return Ok(None);
303 };
304 let document_sofar = self.body_sofar_for_signature().body();
305 let signature_item_line = self.lines.peeked_line(&peeked.line);
306 let signature_item_kw_spc = signature_item_line.strip_end_counted(peeked.args_len);
307 Ok(Some(SignatureHashInputs {
308 body,
309 document_sofar,
310 signature_item_kw_spc,
311 signature_item_line,
312 }))
313 }
314
315 /// Yield the next item.
316 #[allow(clippy::string_slice)] // TODO
317 pub fn next_item(&mut self) -> Result<Option<UnparsedItem<'s>>, EP> {
318 self.peek_internal()?;
319 let peeked = match self.peeked {
320 PeekState::None { .. } => return Ok(None),
321 PeekState::Some { .. } => match mem::replace(
322 &mut self.peeked,
323 PeekState::None {
324 yielded_item_lno: self.lines.peek_lno(),
325 },
326 ) {
327 PeekState::Some(peeked) => peeked,
328 PeekState::None { .. } => panic!("it was Some just now"),
329 },
330 };
331
332 let keyword = peeked.keyword;
333 let line = self.lines.consume_peeked(peeked.line);
334 let args = &line[keyword.len()..];
335 let options = self.options;
336 let args = ArgumentStream::new(args, line.len(), options);
337
338 let object = if self.lines.remaining().starts_with('-') {
339 // Swap out self.lines, so that if we do not find matching delimiters, we don't
340 // ever yield any more items. Otherwise, if we continue reading after an error, we
341 // might get a framing mismatch where we treat base64 contents as if it were item
342 // keyword lines.
343 let leave_if_error = self.lines.clone_entirely_consumed();
344 let mut lines = mem::replace(&mut self.lines, leave_if_error);
345 let self_lines_prevent = &mut self.lines;
346
347 fn pem_delimiter<'s>(lines: &mut Lines<'s>, start: &str) -> Result<&'s str, EP> {
348 let line = lines.next().ok_or(
349 // If this is the *header*, we already know there's a line,
350 // so this error path is only for footers.
351 EP::ObjectMissingFooter,
352 )?;
353 let label = line
354 .strip_prefix(start)
355 .ok_or(EP::InvalidObjectDelimiters)?
356 .strip_suffix(PEM_AFTER_LABEL)
357 .ok_or(EP::InvalidObjectDelimiters)?;
358 Ok(label)
359 }
360
361 let label1 = pem_delimiter(&mut lines, PEM_HEADER_START)?;
362 let base64_start_remaining = lines.remaining();
363 while !lines.remaining().starts_with('-') {
364 let _: &str = lines.next().ok_or(EP::ObjectMissingFooter)?;
365 }
366 let data_b64 = base64_start_remaining.strip_end_counted(lines.remaining().len());
367 let label2 = pem_delimiter(&mut lines, PEM_FOOTER_START)?;
368 let label = [label1, label2]
369 .into_iter()
370 .all_equal_value()
371 .map_err(|_| EP::ObjectMismatchedLabels)?;
372
373 // Proves that self.lines isn't used between setup and here: we have it borrowed.
374 let _: &mut Lines = self_lines_prevent;
375 self.lines = lines;
376
377 Some(UnparsedObject {
378 label,
379 data_b64,
380 options,
381 })
382 } else {
383 None
384 };
385
386 Ok(Some(UnparsedItem {
387 keyword,
388 args,
389 object,
390 }))
391 }
392}
393
394impl<'s> UnparsedItem<'s> {
395 /// Access the arguments, mutably (for consuming and parsing them)
396 pub fn args_mut(&mut self) -> &mut ArgumentStream<'s> {
397 &mut self.args
398 }
399 /// Access a copy of the arguments
400 ///
401 /// When using this, be careful not to process any arguments twice.
402 pub fn args_copy(&self) -> ArgumentStream<'s> {
403 self.args.clone()
404 }
405
406 /// Access the arguments (readonly)
407 ///
408 /// When using this, be careful not to process any arguments twice.
409 pub fn args(&self) -> &ArgumentStream<'s> {
410 &self.args
411 }
412
413 /// Check that this item has no Object.
414 pub fn check_no_object(&self) -> Result<(), EP> {
415 if self.object.is_some() {
416 return Err(EP::ObjectUnexpected);
417 }
418 Ok(())
419 }
420 /// Convenience method for handling an error parsing an argument
421 ///
422 /// Returns a closure that converts every error into [`ArgumentError::Invalid`]
423 /// and then to an [`ErrorProblem`] using
424 /// [`.args().handle_error()`](ArgumentStream::handle_error).
425 ///
426 /// Useful in manual `ItemValueParseable` impls, when parsing arguments ad-hoc.
427 pub fn invalid_argument_handler<E>(
428 &self,
429 field: &'static str,
430 ) -> impl FnOnce(E) -> ErrorProblem {
431 let error = self.args().handle_error(field, AE::Invalid);
432 move |_any_error| error
433 }
434}
435
436#[deprecated = "use types::NoFurtherArguments"]
437pub use crate::types::NoMoreArguments as NoFurtherArguments;
438
439impl<'s> Iterator for ItemStream<'s> {
440 type Item = Result<UnparsedItem<'s>, EP>;
441 fn next(&mut self) -> Option<Result<UnparsedItem<'s>, EP>> {
442 self.next_item().transpose()
443 }
444}
445
446impl<'s> ArgumentStream<'s> {
447 /// Make a new `ArgumentStream` from a string
448 ///
449 /// The string may start with whitespace (which will be ignored).
450 pub fn new(rest: &'s str, whole_line_len: usize, options: &'s ParseOptions) -> Self {
451 let previous_rest_len = whole_line_len;
452 ArgumentStream {
453 rest,
454 whole_line_len,
455 previous_rest_len,
456 options,
457 }
458 }
459
460 /// Consume this whole `ArgumentStream`, giving the remaining arguments as a string
461 ///
462 /// The returned string won't start with whitespace.
463 //
464 /// `self` will be empty on return.
465 // (We don't take `self` by value because that makes use with `UnparsedItem` annoying.)
466 pub fn into_remaining(&mut self) -> &'s str {
467 self.prep_yield();
468 mem::take(&mut self.rest)
469 }
470
471 /// Return the component parts of this `ArgumentStream`
472 ///
473 /// The returned string might start with whitespace.
474 pub fn whole_line_len(&self) -> usize {
475 self.whole_line_len
476 }
477
478 /// Prepares to yield an argument (or the rest)
479 ///
480 /// * Trims leading WS from `rest`.
481 /// * Records the `previous_rest_len`
482 fn prep_yield(&mut self) {
483 self.rest = self.rest.trim_start_matches(WS);
484 self.previous_rest_len = self.rest.len();
485 }
486
487 /// Prepares to yield, and then determines if there *is* anything to yield.
488 ///
489 /// * Trim leading whitespace
490 /// * Records the `previous_rest_len`
491 /// * See if we're now empty
492 pub fn something_to_yield(&mut self) -> bool {
493 self.prep_yield();
494 !self.rest.is_empty()
495 }
496
497 /// Throw and error if there are further arguments
498 //
499 // (We don't take `self` by value because that makes use with `UnparsedItem` annoying.)
500 pub fn reject_extra_args(&mut self) -> Result<NoFurtherArguments, UnexpectedArgument> {
501 if self.something_to_yield() {
502 let column = self.next_arg_column();
503 Err(UnexpectedArgument { column })
504 } else {
505 Ok(NoFurtherArguments)
506 }
507 }
508
509 /// Convert a "length of `rest`" into the corresponding column number.
510 fn arg_column_from_rest_len(&self, rest_len: usize) -> usize {
511 // Can't underflow since rest is always part of the whole.
512 // Can't overflow since that would mean the document was as big as the address space.
513 self.whole_line_len - rest_len + 1
514 }
515
516 /// Obtain the column number of the previously yielded argument.
517 ///
518 /// (After `into_remaining`, gives the column number
519 /// of the start of the returned remaining argument string.)
520 pub fn prev_arg_column(&self) -> usize {
521 self.arg_column_from_rest_len(self.previous_rest_len)
522 }
523
524 /// Obtains the column number of the *next* argument.
525 ///
526 /// Should be called after `something_to_yield`; otherwise the returned value
527 /// may point to whitespace which is going to be skipped.
528 // ^ this possible misuse doesn't seem worth defending against with type-fu,
529 // for a private function with few call sites.
530 fn next_arg_column(&self) -> usize {
531 self.arg_column_from_rest_len(self.rest.len())
532 }
533
534 /// Convert an `ArgumentError` to an `ErrorProblem`.
535 ///
536 /// The caller must supply the field name.
537 pub fn handle_error(&self, field: &'static str, ae: ArgumentError) -> ErrorProblem {
538 self.error_handler(field)(ae)
539 }
540
541 /// Return a converter from `ArgumentError` to `ErrorProblem`.
542 ///
543 /// Useful in `.map_err`.
544 pub fn error_handler(
545 &self,
546 field: &'static str,
547 ) -> impl Fn(ArgumentError) -> ErrorProblem + 'static {
548 let column = self.prev_arg_column();
549 move |ae| match ae {
550 AE::Missing => EP::MissingArgument { field },
551 AE::Invalid => EP::InvalidArgument { field, column },
552 AE::Unexpected => EP::UnexpectedArgument { column },
553 }
554 }
555}
556
557impl<'s> Iterator for ArgumentStream<'s> {
558 type Item = &'s str;
559 fn next(&mut self) -> Option<&'s str> {
560 if !self.something_to_yield() {
561 return None;
562 }
563 let arg;
564 (arg, self.rest) = self.rest.split_once(WS).unwrap_or((self.rest, ""));
565 Some(arg)
566 }
567}
568
569impl<'s> UnparsedObject<'s> {
570 /// Obtain the Object data, as decoded bytes
571 pub fn decode_data(&self) -> Result<Vec<u8>, EP> {
572 crate::parse::tokenize::base64_decode_multiline(self.data_b64)
573 .map_err(|_e| EP::ObjectInvalidBase64)
574 }
575}