tor_hsservice/publish/reactor.rs
1//! The onion service publisher reactor.
2//!
3//! Generates and publishes hidden service descriptors in response to various events.
4//!
5//! [`Reactor::run`] is the entry-point of the reactor. It starts the reactor,
6//! and runs until [`Reactor::run_once`] returns [`ShutdownStatus::Terminate`]
7//! or a fatal error occurs. `ShutdownStatus::Terminate` is returned if
8//! any of the channels the reactor is receiving events from is closed
9//! (i.e. when the senders are dropped).
10//!
11//! ## Publisher status
12//!
13//! The publisher has an internal [`PublishStatus`], distinct from its [`State`],
14//! which is used for onion service status reporting.
15//!
16//! The main loop of the reactor reads the current `PublishStatus` from `publish_status_rx`,
17//! and responds by generating and publishing a new descriptor if needed.
18//!
19//! See [`PublishStatus`] and [`Reactor::publish_status_rx`] for more details.
20//!
21//! ## When do we publish?
22//!
23//! We generate and publish a new descriptor if
24//! * the introduction points have changed
25//! * the onion service configuration has changed in a meaningful way (for example,
26//! if the `restricted_discovery` configuration or its [`Anonymity`](crate::Anonymity)
27//! has changed. See [`OnionServiceConfigPublisherView`]).
28//! * there is a new consensus
29//! * it is time to republish the descriptor (after we upload a descriptor,
30//! we schedule it for republishing at a random time between 60 minutes and 120 minutes
31//! in the future)
32//!
33//! ## Onion service status
34//!
35//! With respect to [`OnionServiceStatus`] reporting,
36//! the following state transitions are possible:
37//!
38//!
39//! ```ignore
40//!
41//! update_publish_status(UploadScheduled|AwaitingIpts|RateLimited)
42//! +---------------------------------------+
43//! | |
44//! | v
45//! | +---------------+
46//! | | Bootstrapping |
47//! | +---------------+
48//! | |
49//! | | uploaded to at least
50//! | not enough HsDir uploads succeeded | some HsDirs from each ring
51//! | +-----------------------------+-----------------------+
52//! | | | |
53//! | | all HsDir uploads succeeded |
54//! | | | |
55//! | v v v
56//! | +---------------------+ +---------+ +---------------------+
57//! | | DegradedUnreachable | | Running | | DegradedReachable |
58//! +----------+ | +---------------------+ +---------+ +---------------------+
59//! | Shutdown |-- | | | |
60//! +----------+ | | | |
61//! | | | |
62//! | | | |
63//! | +---------------------------+------------------------+
64//! | | invalid authorized_clients
65//! | | after handling config change
66//! | |
67//! | v
68//! | run_once() returns an error +--------+
69//! +-------------------------------->| Broken |
70//! +--------+
71//! ```
72//!
73//! We can also transition from `Broken`, `DegradedReachable`, or `DegradedUnreachable`
74//! back to `Bootstrapping` (those transitions were omitted for brevity).
75
76use tor_circmgr::ServiceOnionServiceDirTunnel;
77use tor_config::file_watcher::{
78 self, Event as FileEvent, FileEventReceiver, FileEventSender, FileWatcher, FileWatcherBuilder,
79};
80use tor_config_path::{CfgPath, CfgPathResolver};
81use tor_dirclient::{RequestError, SourceInfo};
82use tor_netdir::{DirEvent, NetDir};
83use tor_persist::state_dir::InstanceRawSubdir;
84use tracing::instrument;
85
86use crate::config::OnionServiceConfigPublisherView;
87use crate::config::restricted_discovery::{
88 DirectoryKeyProviderList, RestrictedDiscoveryConfig, RestrictedDiscoveryKeys,
89};
90use crate::status::{DescUploadRetryError, Problem};
91
92use super::*;
93use derive_more::From;
94use fs_mistrust::anon_home::PathExt;
95
96// TODO-CLIENT-AUTH: perhaps we should add a separate CONFIG_CHANGE_REPUBLISH_DEBOUNCE_INTERVAL
97// for rate-limiting the publish jobs triggered by a change in the config?
98//
99// Currently the descriptor publish tasks triggered by changes in the config
100// are rate-limited via the usual rate limiting mechanism
101// (which rate-limits the uploads for 1m).
102//
103// I think this is OK for now, but we might need to rethink this if it becomes problematic
104// (for example, we might want an even longer rate-limit, or to reset any existing rate-limits
105// each time the config is modified).
106
107/// The upload rate-limiting threshold.
108///
109/// Before initiating an upload, the reactor checks if the last upload was at least
110/// `UPLOAD_RATE_LIM_THRESHOLD` seconds ago. If so, it uploads the descriptor to all HsDirs that
111/// need it. If not, it schedules the upload to happen `UPLOAD_RATE_LIM_THRESHOLD` seconds from the
112/// current time.
113//
114// TODO: We may someday need to tune this value; it was chosen more or less arbitrarily.
115const UPLOAD_RATE_LIM_THRESHOLD: Duration = Duration::from_secs(60);
116
117/// The maximum number of concurrent upload tasks per time period.
118//
119// TODO: this value was arbitrarily chosen and may not be optimal. For now, it
120// will have no effect, since the current number of replicas is far less than
121// this value.
122//
123// The uploads for all TPs happen in parallel. As a result, the actual limit for the maximum
124// number of concurrent upload tasks is multiplied by a number which depends on the TP parameters
125// (currently 2, which means the concurrency limit will, in fact, be 32).
126//
127// We should try to decouple this value from the TP parameters.
128const MAX_CONCURRENT_UPLOADS: usize = 16;
129
130/// The maximum time allowed for uploading a descriptor to a single HSDir,
131/// across all attempts.
132pub(crate) const OVERALL_UPLOAD_TIMEOUT: Duration = Duration::from_secs(5 * 60);
133
134/// A reactor for the HsDir [`Publisher`]
135///
136/// The entrypoint is [`Reactor::run`].
137#[must_use = "If you don't call run() on the reactor, it won't publish any descriptors."]
138pub(super) struct Reactor<R: Runtime, M: Mockable> {
139 /// The immutable, shared inner state.
140 imm: Arc<Immutable<R, M>>,
141 /// A source for new network directories that we use to determine
142 /// our HsDirs.
143 dir_provider: Arc<dyn NetDirProvider>,
144 /// The mutable inner state,
145 inner: Arc<Mutex<Inner>>,
146 /// A channel for receiving IPT change notifications.
147 ipt_watcher: IptsPublisherView,
148 /// A channel for receiving onion service config change notifications.
149 config_rx: watch::Receiver<Arc<OnionServiceConfig>>,
150 /// A channel for receiving restricted discovery key_dirs change notifications.
151 key_dirs_rx: FileEventReceiver,
152 /// A channel for sending restricted discovery key_dirs change notifications.
153 ///
154 /// A copy of this sender is handed out to every `FileWatcher` created.
155 key_dirs_tx: FileEventSender,
156 /// A channel for receiving updates regarding our [`PublishStatus`].
157 ///
158 /// The main loop of the reactor watches for updates on this channel.
159 ///
160 /// When the [`PublishStatus`] changes to [`UploadScheduled`](PublishStatus::UploadScheduled),
161 /// we can start publishing descriptors.
162 ///
163 /// If the [`PublishStatus`] is [`AwaitingIpts`](PublishStatus::AwaitingIpts), publishing is
164 /// paused until we receive a notification on `ipt_watcher` telling us the IPT manager has
165 /// established some introduction points.
166 publish_status_rx: watch::Receiver<PublishStatus>,
167 /// A sender for updating our [`PublishStatus`].
168 ///
169 /// When our [`PublishStatus`] changes to [`UploadScheduled`](PublishStatus::UploadScheduled),
170 /// we can start publishing descriptors.
171 publish_status_tx: watch::Sender<PublishStatus>,
172 /// A channel for sending upload completion notifications.
173 ///
174 /// This channel is polled in the main loop of the reactor.
175 upload_task_complete_rx: mpsc::Receiver<TimePeriodUploadResult>,
176 /// A channel for receiving upload completion notifications.
177 ///
178 /// A copy of this sender is handed to each upload task.
179 upload_task_complete_tx: mpsc::Sender<TimePeriodUploadResult>,
180 /// A sender for notifying any pending upload tasks that the reactor is shutting down.
181 ///
182 /// Receivers can use this channel to find out when reactor is dropped.
183 ///
184 /// This is currently only used in [`upload_for_time_period`](Reactor::upload_for_time_period).
185 /// Any future background tasks can also use this channel to detect if the reactor is dropped.
186 ///
187 /// Closing this channel will cause any pending upload tasks to be dropped.
188 shutdown_tx: broadcast::Sender<Void>,
189 /// Path resolver for configuration files.
190 path_resolver: Arc<CfgPathResolver>,
191 /// Queue on which we receive messages from the [`PowManager`] telling us that a seed has
192 /// rotated and thus we need to republish the descriptor for a particular time period.
193 update_from_pow_manager_rx: mpsc::Receiver<TimePeriod>,
194}
195
196/// The immutable, shared state of the descriptor publisher reactor.
197#[derive(Clone)]
198struct Immutable<R: Runtime, M: Mockable> {
199 /// The runtime.
200 runtime: R,
201 /// Mockable state.
202 ///
203 /// This is used for launching circuits and for obtaining random number generators.
204 mockable: M,
205 /// The service for which we're publishing descriptors.
206 nickname: HsNickname,
207 /// The key manager,
208 keymgr: Arc<KeyMgr>,
209 /// A sender for updating the status of the onion service.
210 status_tx: PublisherStatusSender,
211 /// Proof-of-work state.
212 pow_manager: Arc<PowManager<R>>,
213 /// The directory to dump invalid HS descriptors to.
214 bad_hsdescs_dir: InstanceRawSubdir,
215}
216
217impl<R: Runtime, M: Mockable> Immutable<R, M> {
218 /// Create an [`AesOpeKey`] for generating revision counters for the descriptors associated
219 /// with the specified [`TimePeriod`].
220 ///
221 /// If the onion service is not running in offline mode, the key of the returned `AesOpeKey` is
222 /// the private part of the blinded identity key. Otherwise, the key is the private part of the
223 /// descriptor signing key.
224 ///
225 /// Returns an error if the service is running in offline mode and the descriptor signing
226 /// keypair of the specified `period` is not available.
227 //
228 // TODO (#1194): we don't support "offline" mode (yet), so this always returns an AesOpeKey
229 // built from the blinded id key
230 fn create_ope_key(&self, period: TimePeriod) -> Result<AesOpeKey, FatalError> {
231 let ope_key = match read_blind_id_keypair(&self.keymgr, &self.nickname, period)? {
232 Some(key) => {
233 let key: ed25519::ExpandedKeypair = key.into();
234 key.to_secret_key_bytes()[0..32]
235 .try_into()
236 .expect("Wrong length on slice")
237 }
238 None => {
239 // TODO (#1194): we don't support externally provisioned keys (yet), so this branch
240 // is unreachable (for now).
241 let desc_sign_key_spec =
242 DescSigningKeypairSpecifier::new(self.nickname.clone(), period);
243 let key: ed25519::Keypair = self
244 .keymgr
245 .get::<HsDescSigningKeypair>(&desc_sign_key_spec)?
246 // TODO (#1194): internal! is not the right type for this error (we need an
247 // error type for the case where a hidden service running in offline mode has
248 // run out of its pre-previsioned keys).
249 //
250 // This will be addressed when we add support for offline hs_id mode
251 .ok_or_else(|| {
252 internal!(
253 "identity keys are offline, but descriptor signing key is unavailable?!"
254 )
255 })?
256 .into();
257 key.to_bytes()
258 }
259 };
260
261 Ok(AesOpeKey::from_secret(&ope_key))
262 }
263
264 /// Generate a revision counter for a descriptor associated with the specified
265 /// [`TimePeriod`].
266 ///
267 /// Returns a revision counter generated according to the [encrypted time in period] scheme.
268 ///
269 /// [encrypted time in period]: https://spec.torproject.org/rend-spec/revision-counter-mgt.html#encrypted-time
270 fn generate_revision_counter(
271 &self,
272 params: &HsDirParams,
273 now: SystemTime,
274 ) -> Result<RevisionCounter, FatalError> {
275 // TODO: in the future, we might want to compute ope_key once per time period (as oppposed
276 // to each time we generate a new descriptor), for performance reasons.
277 let ope_key = self.create_ope_key(params.time_period())?;
278
279 // TODO: perhaps this should be moved to a new HsDirParams::offset_within_sr() function
280 let srv_start = params.start_of_shard_rand_period();
281 let offset = params.offset_within_srv_period(now).ok_or_else(|| {
282 internal!(
283 "current wallclock time not within SRV range?! (now={:?}, SRV_start={:?})",
284 now,
285 srv_start
286 )
287 })?;
288 let rev = ope_key.encrypt(offset);
289
290 Ok(RevisionCounter::from(rev))
291 }
292}
293
294/// Mockable state for the descriptor publisher reactor.
295///
296/// This enables us to mock parts of the [`Reactor`] for testing purposes.
297#[async_trait]
298pub(crate) trait Mockable: Clone + Send + Sync + Sized + 'static {
299 /// The type of random number generator.
300 type Rng: rand::Rng + rand::CryptoRng;
301
302 /// The type of client circuit.
303 type Tunnel: MockableDirTunnel;
304
305 /// Return a random number generator.
306 fn thread_rng(&self) -> Self::Rng;
307
308 /// Create a circuit of the specified `kind` to `target`.
309 async fn get_or_launch_hs_dir<T>(
310 &self,
311 netdir: &NetDir,
312 target: T,
313 ) -> Result<Self::Tunnel, tor_circmgr::Error>
314 where
315 T: CircTarget + Send + Sync;
316
317 /// Return an estimate-based value for how long we should allow a single
318 /// directory upload operation to complete.
319 ///
320 /// Includes circuit construction, stream opening, upload, and waiting for a
321 /// response.
322 fn estimate_upload_timeout(&self) -> Duration;
323}
324
325/// Mockable client circuit
326#[async_trait]
327pub(crate) trait MockableDirTunnel: Send + Sync {
328 /// The data stream type.
329 type DataStream: AsyncRead + AsyncWrite + Send + Unpin;
330
331 /// Start a new stream to the last relay in the circuit, using
332 /// a BEGIN_DIR cell.
333 async fn begin_dir_stream(&self) -> Result<Self::DataStream, tor_circmgr::Error>;
334
335 /// Try to get a SourceInfo for this circuit, for using it in a directory request.
336 fn source_info(&self) -> tor_proto::Result<Option<SourceInfo>>;
337}
338
339#[async_trait]
340impl MockableDirTunnel for ServiceOnionServiceDirTunnel {
341 type DataStream = tor_proto::client::stream::DataStream;
342
343 async fn begin_dir_stream(&self) -> Result<Self::DataStream, tor_circmgr::Error> {
344 Self::begin_dir_stream(self).await
345 }
346
347 fn source_info(&self) -> tor_proto::Result<Option<SourceInfo>> {
348 SourceInfo::from_tunnel(self)
349 }
350}
351
352/// The real version of the mockable state of the reactor.
353#[derive(Clone, From, Into)]
354pub(crate) struct Real<R: Runtime>(Arc<HsCircPool<R>>);
355
356#[async_trait]
357impl<R: Runtime> Mockable for Real<R> {
358 type Rng = rand::rngs::ThreadRng;
359 type Tunnel = ServiceOnionServiceDirTunnel;
360
361 fn thread_rng(&self) -> Self::Rng {
362 rand::rng()
363 }
364
365 #[instrument(level = "trace", skip_all)]
366 async fn get_or_launch_hs_dir<T>(
367 &self,
368 netdir: &NetDir,
369 target: T,
370 ) -> Result<Self::Tunnel, tor_circmgr::Error>
371 where
372 T: CircTarget + Send + Sync,
373 {
374 self.0.get_or_launch_svc_dir(netdir, target).await
375 }
376
377 fn estimate_upload_timeout(&self) -> Duration {
378 use tor_circmgr::timeouts::Action;
379 let est_build = self.0.estimate_timeout(&Action::BuildCircuit { length: 4 });
380 let est_roundtrip = self.0.estimate_timeout(&Action::RoundTrip { length: 4 });
381 // We assume that in the worst case we'll have to wait for an entire
382 // circuit construction and two round-trips to the hsdir.
383 let est_total = est_build + est_roundtrip * 2;
384 // We always allow _at least_ this much time, in case our estimate is
385 // ridiculously low.
386 let min_timeout = Duration::from_secs(30);
387 max(est_total, min_timeout)
388 }
389}
390
391/// The mutable state of a [`Reactor`].
392struct Inner {
393 /// The onion service config.
394 config: Arc<OnionServiceConfigPublisherView>,
395 /// Watcher for key_dirs.
396 ///
397 /// Set to `None` if the reactor is not running, or if `watch_configuration` is false.
398 ///
399 /// The watcher is recreated whenever the `restricted_discovery.key_dirs` change.
400 file_watcher: Option<FileWatcher>,
401 /// The relevant time periods.
402 ///
403 /// This includes the current time period, as well as any other time periods we need to be
404 /// publishing descriptors for.
405 ///
406 /// This is empty until we fetch our first netdir in [`Reactor::run`].
407 time_periods: Vec<TimePeriodContext>,
408 /// Our most up to date netdir.
409 ///
410 /// This is initialized in [`Reactor::run`].
411 netdir: Option<Arc<NetDir>>,
412 /// The timestamp of our last upload.
413 ///
414 /// This is the time when the last update was _initiated_ (rather than completed), to prevent
415 /// the publisher from spawning multiple upload tasks at once in response to multiple external
416 /// events happening in quick succession, such as the IPT manager sending multiple IPT change
417 /// notifications in a short time frame (#1142), or an IPT change notification that's
418 /// immediately followed by a consensus change. Starting two upload tasks at once is not only
419 /// inefficient, but it also causes the publisher to generate two different descriptors with
420 /// the same revision counter (the revision counter is derived from the current timestamp),
421 /// which ultimately causes the slower upload task to fail (see #1142).
422 ///
423 /// Note: This is only used for deciding when to reschedule a rate-limited upload. It is _not_
424 /// used for retrying failed uploads (these are handled internally by
425 /// [`Reactor::upload_descriptor_with_retries`]).
426 last_uploaded: Option<Instant>,
427 /// A max-heap containing the time periods for which we need to reupload the descriptor.
428 // TODO: we are currently reuploading more than nececessary.
429 // Ideally, this shouldn't contain contain duplicate TimePeriods,
430 // because we only need to retain the latest reupload time for each time period.
431 //
432 // Currently, if, for some reason, we upload the descriptor multiple times for the same TP,
433 // we will end up with multiple ReuploadTimer entries for that TP,
434 // each of which will (eventually) result in a reupload.
435 //
436 // TODO: maybe this should just be a HashMap<TimePeriod, Instant>
437 //
438 // See https://gitlab.torproject.org/tpo/core/arti/-/merge_requests/1971#note_2994950
439 reupload_timers: BinaryHeap<ReuploadTimer>,
440 /// The restricted discovery authorized clients.
441 ///
442 /// `None`, unless the service is running in restricted discovery mode.
443 authorized_clients: Option<Arc<RestrictedDiscoveryKeys>>,
444}
445
446/// The part of the reactor state that changes with every time period.
447struct TimePeriodContext {
448 /// The HsDir params.
449 params: HsDirParams,
450 /// The HsDirs to use in this time period.
451 ///
452 // We keep a list of `RelayIds` because we can't store a `Relay<'_>` inside the reactor
453 // (the lifetime of a relay is tied to the lifetime of its corresponding `NetDir`. To
454 // store `Relay<'_>`s in the reactor, we'd need a way of atomically swapping out both the
455 // `NetDir` and the cached relays, and to convince Rust what we're doing is sound)
456 hs_dirs: Vec<(RelayIds, DescriptorStatus)>,
457 /// The revision counter of the last successful upload, if any.
458 last_successful: Option<RevisionCounter>,
459 /// The outcome of the last upload, if any.
460 upload_results: Vec<HsDirUploadStatus>,
461}
462
463impl TimePeriodContext {
464 /// Create a new `TimePeriodContext`.
465 ///
466 /// Any of the specified `old_hsdirs` also present in the new list of HsDirs
467 /// (returned by `NetDir::hs_dirs_upload`) will have their `DescriptorStatus` preserved.
468 fn new<'r>(
469 params: HsDirParams,
470 blind_id: HsBlindId,
471 netdir: &Arc<NetDir>,
472 old_hsdirs: impl Iterator<Item = &'r (RelayIds, DescriptorStatus)>,
473 old_upload_results: Vec<HsDirUploadStatus>,
474 ) -> Result<Self, FatalError> {
475 let period = params.time_period();
476 let hs_dirs = Self::compute_hsdirs(period, blind_id, netdir, old_hsdirs)?;
477 let upload_results = old_upload_results
478 .into_iter()
479 .filter(|res|
480 // Check if the HsDir of this result still exists
481 hs_dirs
482 .iter()
483 .any(|(relay_ids, _status)| relay_ids == &res.relay_ids))
484 .collect();
485
486 Ok(Self {
487 params,
488 hs_dirs,
489 last_successful: None,
490 upload_results,
491 })
492 }
493
494 /// Recompute the HsDirs for this time period.
495 fn compute_hsdirs<'r>(
496 period: TimePeriod,
497 blind_id: HsBlindId,
498 netdir: &Arc<NetDir>,
499 mut old_hsdirs: impl Iterator<Item = &'r (RelayIds, DescriptorStatus)>,
500 ) -> Result<Vec<(RelayIds, DescriptorStatus)>, FatalError> {
501 let hs_dirs = netdir.hs_dirs_upload(blind_id, period)?;
502
503 Ok(hs_dirs
504 .map(|hs_dir| {
505 let mut builder = RelayIds::builder();
506 if let Some(ed_id) = hs_dir.ed_identity() {
507 builder.ed_identity(*ed_id);
508 }
509
510 if let Some(rsa_id) = hs_dir.rsa_identity() {
511 builder.rsa_identity(*rsa_id);
512 }
513
514 let relay_id = builder.build().unwrap_or_else(|_| RelayIds::empty());
515
516 // Have we uploaded the descriptor to thiw relay before? If so, we don't need to
517 // reupload it unless it was already dirty and due for a reupload.
518 let status = match old_hsdirs.find(|(id, _)| *id == relay_id) {
519 Some((_, status)) => *status,
520 None => DescriptorStatus::Dirty,
521 };
522
523 (relay_id, status)
524 })
525 .collect::<Vec<_>>())
526 }
527
528 /// Mark the descriptor dirty for all HSDirs of this time period.
529 fn mark_all_dirty(&mut self) {
530 self.hs_dirs
531 .iter_mut()
532 .for_each(|(_relay_id, status)| *status = DescriptorStatus::Dirty);
533 }
534
535 /// Update the upload result for this time period.
536 fn set_upload_results(&mut self, upload_results: Vec<HsDirUploadStatus>) {
537 self.upload_results = upload_results;
538 }
539}
540
541/// An error that occurs while trying to upload a descriptor.
542#[derive(Clone, Debug, thiserror::Error)]
543#[non_exhaustive]
544pub enum UploadError {
545 /// An error that has occurred after we have contacted a directory cache and made a circuit to it.
546 #[error("descriptor upload request failed: {}", _0.error)]
547 Request(#[from] RequestFailedError),
548
549 /// Failed to establish circuit to hidden service directory
550 #[error("could not build circuit to HsDir")]
551 Circuit(#[from] tor_circmgr::Error),
552
553 /// Failed to establish stream to hidden service directory
554 #[error("failed to establish directory stream to HsDir")]
555 Stream(#[source] tor_circmgr::Error),
556
557 /// An internal error.
558 #[error("Internal error")]
559 Bug(#[from] tor_error::Bug),
560}
561define_asref_dyn_std_error!(UploadError);
562
563impl UploadError {
564 /// Return true if this error is one that we should report as a suspicious event,
565 /// along with the dirserver, and description of the relevant document.
566 pub(crate) fn should_report_as_suspicious(&self) -> bool {
567 match self {
568 UploadError::Request(e) => e.error.should_report_as_suspicious_if_anon(),
569 UploadError::Circuit(_) => false, // TODO prop360
570 UploadError::Stream(_) => false, // TODO prop360
571 UploadError::Bug(_) => false,
572 }
573 }
574}
575
576impl<R: Runtime, M: Mockable> Reactor<R, M> {
577 /// Create a new `Reactor`.
578 #[allow(clippy::too_many_arguments)]
579 pub(super) fn new(
580 runtime: R,
581 nickname: HsNickname,
582 dir_provider: Arc<dyn NetDirProvider>,
583 mockable: M,
584 config: &OnionServiceConfig,
585 ipt_watcher: IptsPublisherView,
586 config_rx: watch::Receiver<Arc<OnionServiceConfig>>,
587 status_tx: PublisherStatusSender,
588 keymgr: Arc<KeyMgr>,
589 path_resolver: Arc<CfgPathResolver>,
590 pow_manager: Arc<PowManager<R>>,
591 update_from_pow_manager_rx: mpsc::Receiver<TimePeriod>,
592 bad_hsdescs_dir: InstanceRawSubdir,
593 ) -> Self {
594 /// The maximum size of the upload completion notifier channel.
595 ///
596 /// The channel we use this for is a futures::mpsc channel, which has a capacity of
597 /// `UPLOAD_CHAN_BUF_SIZE + num-senders`. We don't need the buffer size to be non-zero, as
598 /// each sender will send exactly one message.
599 const UPLOAD_CHAN_BUF_SIZE: usize = 0;
600
601 // Internally-generated instructions, no need for mq.
602 let (upload_task_complete_tx, upload_task_complete_rx) =
603 mpsc_channel_no_memquota(UPLOAD_CHAN_BUF_SIZE);
604
605 let (publish_status_tx, publish_status_rx) = watch::channel();
606 // Setting the buffer size to zero here is OK,
607 // since we never actually send anything on this channel.
608 let (shutdown_tx, _shutdown_rx) = broadcast::channel(0);
609
610 let authorized_clients =
611 Self::read_authorized_clients(&config.restricted_discovery, &path_resolver);
612
613 // Create a channel for watching for changes in the configured
614 // restricted_discovery.key_dirs.
615 let (key_dirs_tx, key_dirs_rx) = file_watcher::channel();
616
617 let imm = Immutable {
618 runtime,
619 mockable,
620 nickname,
621 keymgr,
622 status_tx,
623 pow_manager,
624 bad_hsdescs_dir,
625 };
626
627 let inner = Inner {
628 time_periods: vec![],
629 config: Arc::new(config.into()),
630 file_watcher: None,
631 netdir: None,
632 last_uploaded: None,
633 reupload_timers: Default::default(),
634 authorized_clients,
635 };
636
637 Self {
638 imm: Arc::new(imm),
639 inner: Arc::new(Mutex::new(inner)),
640 dir_provider,
641 ipt_watcher,
642 config_rx,
643 key_dirs_rx,
644 key_dirs_tx,
645 publish_status_rx,
646 publish_status_tx,
647 upload_task_complete_rx,
648 upload_task_complete_tx,
649 shutdown_tx,
650 path_resolver,
651 update_from_pow_manager_rx,
652 }
653 }
654
655 /// Start the reactor.
656 ///
657 /// Under normal circumstances, this function runs indefinitely.
658 ///
659 /// Note: this also spawns the "reminder task" that we use to reschedule uploads whenever an
660 /// upload fails or is rate-limited.
661 pub(super) async fn run(mut self) -> Result<(), FatalError> {
662 debug!(nickname=%self.imm.nickname, "starting descriptor publisher reactor");
663
664 {
665 let netdir = self
666 .dir_provider
667 .wait_for_netdir(Timeliness::Timely)
668 .await?;
669 let time_periods = self.compute_time_periods(&netdir, &[])?;
670
671 let mut inner = self.inner.lock().expect("poisoned lock");
672
673 inner.netdir = Some(netdir);
674 inner.time_periods = time_periods;
675 }
676
677 // Create the initial key_dirs watcher.
678 self.update_file_watcher();
679
680 loop {
681 match self.run_once().await {
682 Ok(ShutdownStatus::Continue) => continue,
683 Ok(ShutdownStatus::Terminate) => {
684 debug!(nickname=%self.imm.nickname, "descriptor publisher is shutting down!");
685
686 self.imm.status_tx.send_shutdown();
687 return Ok(());
688 }
689 Err(e) => {
690 error_report!(
691 e,
692 "HS service {}: descriptor publisher crashed!",
693 self.imm.nickname
694 );
695
696 self.imm.status_tx.send_broken(e.clone());
697
698 return Err(e);
699 }
700 }
701 }
702 }
703
704 /// Run one iteration of the reactor loop.
705 async fn run_once(&mut self) -> Result<ShutdownStatus, FatalError> {
706 let mut netdir_events = self.dir_provider.events();
707
708 // Note: TrackingNow tracks the values it is compared with.
709 // This is equivalent to sleeping for (until - now) units of time,
710 let upload_rate_lim: TrackingNow = TrackingNow::now(&self.imm.runtime);
711 if let PublishStatus::RateLimited(until) = self.status() {
712 if upload_rate_lim > until {
713 // We are no longer rate-limited
714 self.expire_rate_limit().await?;
715 }
716 }
717
718 let reupload_tracking = TrackingNow::now(&self.imm.runtime);
719 let mut reupload_periods = vec![];
720 {
721 let mut inner = self.inner.lock().expect("poisoned lock");
722 let inner = &mut *inner;
723 while let Some(reupload) = inner.reupload_timers.peek().copied() {
724 // First, extract all the timeouts that already elapsed.
725 if reupload.when <= reupload_tracking {
726 inner.reupload_timers.pop();
727 reupload_periods.push(reupload.period);
728 } else {
729 // We are not ready to schedule any more reuploads.
730 //
731 // How much we need to sleep is implicitly
732 // tracked in reupload_tracking (through
733 // the TrackingNow implementation)
734 break;
735 }
736 }
737 }
738
739 // Check if it's time to schedule any reuploads.
740 for period in reupload_periods {
741 if self.mark_dirty(&period) {
742 debug!(
743 time_period=?period,
744 "descriptor reupload timer elapsed; scheduling reupload",
745 );
746 self.update_publish_status_unless_rate_lim(PublishStatus::UploadScheduled)
747 .await?;
748 }
749 }
750
751 select_biased! {
752 res = self.upload_task_complete_rx.next().fuse() => {
753 let Some(upload_res) = res else {
754 return Ok(ShutdownStatus::Terminate);
755 };
756
757 self.handle_upload_results(upload_res);
758 self.upload_result_to_svc_status()?;
759 },
760 () = upload_rate_lim.wait_for_earliest(&self.imm.runtime).fuse() => {
761 self.expire_rate_limit().await?;
762 },
763 () = reupload_tracking.wait_for_earliest(&self.imm.runtime).fuse() => {
764 // Run another iteration, executing run_once again. This time, we will remove the
765 // expired reupload from self.reupload_timers, mark the descriptor dirty for all
766 // relevant HsDirs, and schedule the upload by setting our status to
767 // UploadScheduled.
768 return Ok(ShutdownStatus::Continue);
769 },
770 netdir_event = netdir_events.next().fuse() => {
771 let Some(netdir_event) = netdir_event else {
772 debug!("netdir event stream ended");
773 return Ok(ShutdownStatus::Terminate);
774 };
775
776 if !matches!(netdir_event, DirEvent::NewConsensus) {
777 return Ok(ShutdownStatus::Continue);
778 };
779
780 // The consensus changed. Grab a new NetDir.
781 let netdir = match self.dir_provider.netdir(Timeliness::Timely) {
782 Ok(y) => y,
783 Err(e) => {
784 error_report!(e, "HS service {}: netdir unavailable. Retrying...", self.imm.nickname);
785 // Hopefully a netdir will appear in the future.
786 // in the meantime, suspend operations.
787 //
788 // TODO (#1218): there is a bug here: we stop reading on our inputs
789 // including eg publish_status_rx, but it is our job to log some of
790 // these things. While we are waiting for a netdir, all those messages
791 // are "stuck"; they'll appear later, with misleading timestamps.
792 //
793 // Probably this should be fixed by moving the logging
794 // out of the reactor, where it won't be blocked.
795 self.dir_provider.wait_for_netdir(Timeliness::Timely)
796 .await?
797 }
798 };
799 let relevant_periods = netdir.hs_all_time_periods();
800 self.handle_consensus_change(netdir).await?;
801 expire_publisher_keys(
802 &self.imm.keymgr,
803 &self.imm.nickname,
804 &relevant_periods,
805 ).unwrap_or_else(|e| {
806 error_report!(e, "failed to remove expired keys");
807 });
808 }
809 update = self.ipt_watcher.await_update().fuse() => {
810 if self.handle_ipt_change(update).await? == ShutdownStatus::Terminate {
811 return Ok(ShutdownStatus::Terminate);
812 }
813 },
814 config = self.config_rx.next().fuse() => {
815 let Some(config) = config else {
816 return Ok(ShutdownStatus::Terminate);
817 };
818
819 self.handle_svc_config_change(&config).await?;
820 },
821 res = self.key_dirs_rx.next().fuse() => {
822 let Some(event) = res else {
823 return Ok(ShutdownStatus::Terminate);
824 };
825
826 while let Some(_ignore) = self.key_dirs_rx.try_recv() {
827 // Discard other events, so that we only reload once.
828 }
829
830 self.handle_key_dirs_change(event).await?;
831 }
832 should_upload = self.publish_status_rx.next().fuse() => {
833 let Some(should_upload) = should_upload else {
834 return Ok(ShutdownStatus::Terminate);
835 };
836
837 // Our PublishStatus changed -- are we ready to publish?
838 if should_upload == PublishStatus::UploadScheduled {
839 self.update_publish_status_unless_waiting(PublishStatus::Idle).await?;
840 self.upload_all().await?;
841 }
842 }
843 update_tp_pow_seed = self.update_from_pow_manager_rx.next().fuse() => {
844 debug!("Update PoW seed for TP!");
845 let Some(time_period) = update_tp_pow_seed else {
846 return Ok(ShutdownStatus::Terminate);
847 };
848 self.mark_dirty(&time_period);
849 self.upload_all().await?;
850 }
851 }
852
853 Ok(ShutdownStatus::Continue)
854 }
855
856 /// Returns the current status of the publisher
857 fn status(&self) -> PublishStatus {
858 *self.publish_status_rx.borrow()
859 }
860
861 /// Handle a batch of upload outcomes,
862 /// possibly updating the status of the descriptor for the corresponding HSDirs.
863 fn handle_upload_results(&self, results: TimePeriodUploadResult) {
864 let mut inner = self.inner.lock().expect("poisoned lock");
865 let inner = &mut *inner;
866
867 // Check which time period these uploads pertain to.
868 let period = inner
869 .time_periods
870 .iter_mut()
871 .find(|ctx| ctx.params.time_period() == results.time_period);
872
873 let Some(period) = period else {
874 // The uploads were for a time period that is no longer relevant, so we
875 // can ignore the result.
876 return;
877 };
878
879 // We will need to reupload this descriptor at some point, so we pick
880 // a random time between 60 minutes and 120 minutes in the future.
881 //
882 // See https://spec.torproject.org/rend-spec/deriving-keys.html#WHEN-HSDESC
883 let mut rng = self.imm.mockable.thread_rng();
884 // TODO SPEC: Control republish period using a consensus parameter?
885 let minutes = rng.gen_range_checked(60..=120).expect("low > high?!");
886 let duration = Duration::from_secs(minutes * 60);
887 let reupload_when = self.imm.runtime.now() + duration;
888 let time_period = period.params.time_period();
889
890 info!(
891 time_period=?time_period,
892 "reuploading descriptor in {}",
893 humantime::format_duration(duration),
894 );
895
896 inner.reupload_timers.push(ReuploadTimer {
897 period: time_period,
898 when: reupload_when,
899 });
900
901 let mut upload_results = vec![];
902 for upload_res in results.hsdir_result {
903 let relay = period
904 .hs_dirs
905 .iter_mut()
906 .find(|(relay_ids, _status)| relay_ids == &upload_res.relay_ids);
907
908 let Some((_relay, status)): Option<&mut (RelayIds, _)> = relay else {
909 // This HSDir went away, so the result doesn't matter.
910 // Continue processing the rest of the results
911 continue;
912 };
913
914 if upload_res.upload_res.is_ok() {
915 let update_last_successful = match period.last_successful {
916 None => true,
917 Some(counter) => counter <= upload_res.revision_counter,
918 };
919
920 if update_last_successful {
921 period.last_successful = Some(upload_res.revision_counter);
922 // TODO (#1098): Is it possible that this won't update the statuses promptly
923 // enough. For example, it's possible for the reactor to see a Dirty descriptor
924 // and start an upload task for a descriptor has already been uploaded (or is
925 // being uploaded) in another task, but whose upload results have not yet been
926 // processed.
927 //
928 // This is probably made worse by the fact that the statuses are updated in
929 // batches (grouped by time period), rather than one by one as the upload tasks
930 // complete (updating the status involves locking the inner mutex, and I wanted
931 // to minimize the locking/unlocking overheads). I'm not sure handling the
932 // updates in batches was the correct decision here.
933 *status = DescriptorStatus::Clean;
934 }
935 }
936
937 upload_results.push(upload_res);
938 }
939
940 period.set_upload_results(upload_results);
941 }
942
943 /// Maybe update our list of HsDirs.
944 async fn handle_consensus_change(&mut self, netdir: Arc<NetDir>) -> Result<(), FatalError> {
945 trace!("the consensus has changed; recomputing HSDirs");
946
947 let _old: Option<Arc<NetDir>> = self.replace_netdir(netdir);
948
949 self.recompute_hs_dirs()?;
950 self.update_publish_status_unless_waiting(PublishStatus::UploadScheduled)
951 .await?;
952
953 // If the time period has changed, some of our upload results may now be irrelevant,
954 // so we might need to update our status (for example, if our uploads are
955 // for a no-longer-relevant time period, it means we might be able to update
956 // out status from "degraded" to "running")
957 self.upload_result_to_svc_status()?;
958
959 Ok(())
960 }
961
962 /// Recompute the HsDirs for all relevant time periods.
963 fn recompute_hs_dirs(&self) -> Result<(), FatalError> {
964 let mut inner = self.inner.lock().expect("poisoned lock");
965 let inner = &mut *inner;
966
967 let netdir = Arc::clone(
968 inner
969 .netdir
970 .as_ref()
971 .ok_or_else(|| internal!("started upload task without a netdir"))?,
972 );
973
974 // Update our list of relevant time periods.
975 let new_time_periods = self.compute_time_periods(&netdir, &inner.time_periods)?;
976 inner.time_periods = new_time_periods;
977
978 Ok(())
979 }
980
981 /// Compute the [`TimePeriodContext`]s for the time periods from the specified [`NetDir`].
982 ///
983 /// The specified `time_periods` are used to preserve the `DescriptorStatus` of the
984 /// HsDirs where possible.
985 fn compute_time_periods(
986 &self,
987 netdir: &Arc<NetDir>,
988 time_periods: &[TimePeriodContext],
989 ) -> Result<Vec<TimePeriodContext>, FatalError> {
990 netdir
991 .hs_all_time_periods()
992 .iter()
993 .map(|params| {
994 let period = params.time_period();
995 let blind_id_kp =
996 read_blind_id_keypair(&self.imm.keymgr, &self.imm.nickname, period)?
997 // Note: for now, read_blind_id_keypair cannot return Ok(None).
998 // It's supposed to return Ok(None) if we're in offline hsid mode,
999 // but that might change when we do #1194
1000 .ok_or_else(|| internal!("offline hsid mode not supported"))?;
1001
1002 let blind_id: HsBlindIdKey = (&blind_id_kp).into();
1003
1004 // If our previous `TimePeriodContext`s also had an entry for `period`, we need to
1005 // preserve the `DescriptorStatus` of its HsDirs. This helps prevent unnecessarily
1006 // publishing the descriptor to the HsDirs that already have it (the ones that are
1007 // marked with DescriptorStatus::Clean).
1008 //
1009 // In other words, we only want to publish to those HsDirs that
1010 // * are part of a new time period (which we have never published the descriptor
1011 // for), or
1012 // * have just been added to the ring of a time period we already knew about
1013 if let Some(ctx) = time_periods
1014 .iter()
1015 .find(|ctx| ctx.params.time_period() == period)
1016 {
1017 TimePeriodContext::new(
1018 params.clone(),
1019 blind_id.into(),
1020 netdir,
1021 ctx.hs_dirs.iter(),
1022 ctx.upload_results.clone(),
1023 )
1024 } else {
1025 // Passing an empty iterator here means all HsDirs in this TimePeriodContext
1026 // will be marked as dirty, meaning we will need to upload our descriptor to them.
1027 TimePeriodContext::new(
1028 params.clone(),
1029 blind_id.into(),
1030 netdir,
1031 iter::empty(),
1032 vec![],
1033 )
1034 }
1035 })
1036 .collect::<Result<Vec<TimePeriodContext>, FatalError>>()
1037 }
1038
1039 /// Replace the old netdir with the new, returning the old.
1040 fn replace_netdir(&self, new_netdir: Arc<NetDir>) -> Option<Arc<NetDir>> {
1041 self.inner
1042 .lock()
1043 .expect("poisoned lock")
1044 .netdir
1045 .replace(new_netdir)
1046 }
1047
1048 /// Replace our view of the service config with `new_config` if `new_config` contains changes
1049 /// that would cause us to generate a new descriptor.
1050 fn replace_config_if_changed(&self, new_config: Arc<OnionServiceConfigPublisherView>) -> bool {
1051 let mut inner = self.inner.lock().expect("poisoned lock");
1052 let old_config = &mut inner.config;
1053
1054 // The fields we're interested in haven't changed, so there's no need to update
1055 // `inner.config`.
1056 if *old_config == new_config {
1057 return false;
1058 }
1059
1060 let log_change = match (
1061 old_config.restricted_discovery.enabled,
1062 new_config.restricted_discovery.enabled,
1063 ) {
1064 (true, false) => Some("Disabling restricted discovery mode"),
1065 (false, true) => Some("Enabling restricted discovery mode"),
1066 _ => None,
1067 };
1068
1069 if let Some(msg) = log_change {
1070 info!(nickname=%self.imm.nickname, "{}", msg);
1071 }
1072
1073 let _old: Arc<OnionServiceConfigPublisherView> = std::mem::replace(old_config, new_config);
1074
1075 true
1076 }
1077
1078 /// Recreate the FileWatcher for watching the restricted discovery key_dirs.
1079 fn update_file_watcher(&self) {
1080 let mut inner = self.inner.lock().expect("poisoned lock");
1081 if inner.config.restricted_discovery.watch_configuration() {
1082 debug!("The restricted_discovery.key_dirs have changed, updating file watcher");
1083 let mut watcher = FileWatcher::builder(self.imm.runtime.clone());
1084
1085 let dirs = inner.config.restricted_discovery.key_dirs().clone();
1086
1087 watch_dirs(&mut watcher, &dirs, &self.path_resolver);
1088
1089 let watcher = watcher
1090 .start_watching(self.key_dirs_tx.clone())
1091 .map_err(|e| {
1092 // TODO: update the publish status (see also the module-level TODO about this).
1093 error_report!(e, "Cannot set file watcher");
1094 })
1095 .ok();
1096 inner.file_watcher = watcher;
1097 } else {
1098 if inner.file_watcher.is_some() {
1099 debug!("removing key_dirs watcher");
1100 }
1101 inner.file_watcher = None;
1102 }
1103 }
1104
1105 /// Read the intro points from `ipt_watcher`, and decide whether we're ready to start
1106 /// uploading.
1107 fn note_ipt_change(&self) -> PublishStatus {
1108 let mut ipts = self.ipt_watcher.borrow_for_publish();
1109 match ipts.ipts.as_mut() {
1110 Some(_ipts) => PublishStatus::UploadScheduled,
1111 None => PublishStatus::AwaitingIpts,
1112 }
1113 }
1114
1115 /// Update our list of introduction points.
1116 async fn handle_ipt_change(
1117 &mut self,
1118 update: Option<Result<(), crate::FatalError>>,
1119 ) -> Result<ShutdownStatus, FatalError> {
1120 trace!(nickname=%self.imm.nickname, "received IPT change notification from IPT manager");
1121 match update {
1122 Some(Ok(())) => {
1123 let should_upload = self.note_ipt_change();
1124 debug!(nickname=%self.imm.nickname, "the introduction points have changed");
1125
1126 self.mark_all_dirty();
1127 self.update_publish_status_unless_rate_lim(should_upload)
1128 .await?;
1129 Ok(ShutdownStatus::Continue)
1130 }
1131 Some(Err(e)) => Err(e),
1132 None => {
1133 debug!(nickname=%self.imm.nickname, "received shut down signal from IPT manager");
1134 Ok(ShutdownStatus::Terminate)
1135 }
1136 }
1137 }
1138
1139 /// Update the `PublishStatus` of the reactor with `new_state`,
1140 /// unless the current state is `AwaitingIpts`.
1141 async fn update_publish_status_unless_waiting(
1142 &mut self,
1143 new_state: PublishStatus,
1144 ) -> Result<(), FatalError> {
1145 // Only update the state if we're not waiting for intro points.
1146 if self.status() != PublishStatus::AwaitingIpts {
1147 self.update_publish_status(new_state).await?;
1148 }
1149
1150 Ok(())
1151 }
1152
1153 /// Update the `PublishStatus` of the reactor with `new_state`,
1154 /// unless the current state is `RateLimited`.
1155 async fn update_publish_status_unless_rate_lim(
1156 &mut self,
1157 new_state: PublishStatus,
1158 ) -> Result<(), FatalError> {
1159 // We can't exit this state until the rate-limit expires.
1160 if !matches!(self.status(), PublishStatus::RateLimited(_)) {
1161 self.update_publish_status(new_state).await?;
1162 }
1163
1164 Ok(())
1165 }
1166
1167 /// Unconditionally update the `PublishStatus` of the reactor with `new_state`.
1168 async fn update_publish_status(&mut self, new_state: PublishStatus) -> Result<(), Bug> {
1169 let onion_status = match new_state {
1170 PublishStatus::Idle => None,
1171 PublishStatus::UploadScheduled
1172 | PublishStatus::AwaitingIpts
1173 | PublishStatus::RateLimited(_) => Some(State::Bootstrapping),
1174 };
1175
1176 if let Some(onion_status) = onion_status {
1177 self.imm.status_tx.send(onion_status, None);
1178 }
1179
1180 trace!(
1181 "publisher reactor status change: {:?} -> {:?}",
1182 self.status(),
1183 new_state
1184 );
1185
1186 self.publish_status_tx.send(new_state).await.map_err(
1187 |_: postage::sink::SendError<_>| internal!("failed to send upload notification?!"),
1188 )?;
1189
1190 Ok(())
1191 }
1192
1193 /// Update the onion svc status based on the results of the last descriptor uploads.
1194 fn upload_result_to_svc_status(&self) -> Result<(), FatalError> {
1195 let inner = self.inner.lock().expect("poisoned lock");
1196 let netdir = inner
1197 .netdir
1198 .as_ref()
1199 .ok_or_else(|| internal!("handling upload results without netdir?!"))?;
1200
1201 let (state, err) = upload_result_state(netdir, &inner.time_periods);
1202 self.imm.status_tx.send(state, err);
1203
1204 Ok(())
1205 }
1206
1207 /// Update the descriptors based on the config change.
1208 async fn handle_svc_config_change(
1209 &mut self,
1210 config: &OnionServiceConfig,
1211 ) -> Result<(), FatalError> {
1212 let new_config = Arc::new(config.into());
1213 if self.replace_config_if_changed(Arc::clone(&new_config)) {
1214 self.update_file_watcher();
1215 self.update_authorized_clients_if_changed();
1216
1217 info!(nickname=%self.imm.nickname, "Config has changed, generating a new descriptor");
1218 self.mark_all_dirty();
1219
1220 // Schedule an upload, unless we're still waiting for IPTs.
1221 self.update_publish_status_unless_waiting(PublishStatus::UploadScheduled)
1222 .await?;
1223 }
1224
1225 Ok(())
1226 }
1227
1228 /// Update the descriptors based on a restricted discovery key_dirs change.
1229 ///
1230 /// If the authorized clients from the [`RestrictedDiscoveryConfig`] have changed,
1231 /// this marks the descriptor as dirty for all time periods,
1232 /// and schedules a reupload.
1233 async fn handle_key_dirs_change(&mut self, event: FileEvent) -> Result<(), FatalError> {
1234 debug!("The configured key_dirs have changed");
1235 match event {
1236 FileEvent::Rescan | FileEvent::FileChanged => {
1237 // These events are handled in the same way, by re-reading the keys from disk
1238 // and republishing the descriptor if necessary
1239 }
1240 _ => return Err(internal!("file watcher event {event:?}").into()),
1241 };
1242
1243 // Update the file watcher, in case the change was triggered by a key_dir move.
1244 self.update_file_watcher();
1245
1246 if self.update_authorized_clients_if_changed() {
1247 self.mark_all_dirty();
1248
1249 // Schedule an upload, unless we're still waiting for IPTs.
1250 self.update_publish_status_unless_waiting(PublishStatus::UploadScheduled)
1251 .await?;
1252 }
1253
1254 Ok(())
1255 }
1256
1257 /// Recreate the authorized_clients based on the current config.
1258 ///
1259 /// Returns `true` if the authorized clients have changed.
1260 fn update_authorized_clients_if_changed(&mut self) -> bool {
1261 let mut inner = self.inner.lock().expect("poisoned lock");
1262 let authorized_clients =
1263 Self::read_authorized_clients(&inner.config.restricted_discovery, &self.path_resolver);
1264
1265 let clients = &mut inner.authorized_clients;
1266 let changed = clients.as_ref() != authorized_clients.as_ref();
1267
1268 if changed {
1269 info!("The restricted discovery mode authorized clients have changed");
1270 *clients = authorized_clients;
1271 }
1272
1273 changed
1274 }
1275
1276 /// Read the authorized `RestrictedDiscoveryKeys` from `config`.
1277 fn read_authorized_clients(
1278 config: &RestrictedDiscoveryConfig,
1279 path_resolver: &CfgPathResolver,
1280 ) -> Option<Arc<RestrictedDiscoveryKeys>> {
1281 let authorized_clients = config.read_keys(path_resolver);
1282
1283 if matches!(authorized_clients.as_ref(), Some(c) if c.is_empty()) {
1284 warn!(
1285 "Running in restricted discovery mode, but we have no authorized clients. Service will be unreachable"
1286 );
1287 }
1288
1289 authorized_clients.map(Arc::new)
1290 }
1291
1292 /// Mark the descriptor dirty for all time periods.
1293 fn mark_all_dirty(&self) {
1294 trace!("marking the descriptor dirty for all time periods");
1295
1296 self.inner
1297 .lock()
1298 .expect("poisoned lock")
1299 .time_periods
1300 .iter_mut()
1301 .for_each(|tp| tp.mark_all_dirty());
1302 }
1303
1304 /// Mark the descriptor dirty for the specified time period.
1305 ///
1306 /// Returns `true` if the specified period is still relevant, and `false` otherwise.
1307 fn mark_dirty(&self, period: &TimePeriod) -> bool {
1308 let mut inner = self.inner.lock().expect("poisoned lock");
1309 let period_ctx = inner
1310 .time_periods
1311 .iter_mut()
1312 .find(|tp| tp.params.time_period() == *period);
1313
1314 match period_ctx {
1315 Some(ctx) => {
1316 trace!(time_period=?period, "marking the descriptor dirty");
1317 ctx.mark_all_dirty();
1318 true
1319 }
1320 None => false,
1321 }
1322 }
1323
1324 /// Try to upload our descriptor to the HsDirs that need it.
1325 ///
1326 /// If we've recently uploaded some descriptors, we return immediately and schedule the upload
1327 /// to happen after [`UPLOAD_RATE_LIM_THRESHOLD`].
1328 ///
1329 /// Failed uploads are retried
1330 /// (see [`upload_descriptor_with_retries`](Reactor::upload_descriptor_with_retries)).
1331 ///
1332 /// If restricted discovery mode is enabled and there are no authorized clients,
1333 /// we abort the upload and set our status to [`State::Broken`].
1334 //
1335 // Note: a broken restricted discovery config won't prevent future uploads from being scheduled
1336 // (for example if the IPTs change),
1337 // which can can cause the publisher's status to oscillate between `Bootstrapping` and `Broken`.
1338 // TODO: we might wish to refactor the publisher to be more sophisticated about this.
1339 //
1340 /// For each current time period, we spawn a task that uploads the descriptor to
1341 /// all the HsDirs on the HsDir ring of that time period.
1342 /// Each task shuts down on completion, or when the reactor is dropped.
1343 ///
1344 /// Each task reports its upload results (`TimePeriodUploadResult`)
1345 /// via the `upload_task_complete_tx` channel.
1346 /// The results are received and processed in the main loop of the reactor.
1347 ///
1348 /// Returns an error if it fails to spawn a task, or if an internal error occurs.
1349 async fn upload_all(&mut self) -> Result<(), FatalError> {
1350 trace!("starting descriptor upload task...");
1351
1352 // Abort the upload entirely if we have an empty list of authorized clients
1353 let authorized_clients = match self.authorized_clients() {
1354 Ok(authorized_clients) => authorized_clients,
1355 Err(e) => {
1356 error_report!(e, "aborting upload");
1357 self.imm.status_tx.send_broken(e.clone());
1358
1359 // Returning an error would shut down the reactor, so we have to return Ok here.
1360 return Ok(());
1361 }
1362 };
1363
1364 let last_uploaded = self.inner.lock().expect("poisoned lock").last_uploaded;
1365 let now = self.imm.runtime.now();
1366 // Check if we should rate-limit this upload.
1367 if let Some(ts) = last_uploaded {
1368 let duration_since_upload = now.duration_since(ts);
1369
1370 if duration_since_upload < UPLOAD_RATE_LIM_THRESHOLD {
1371 return Ok(self.start_rate_limit(UPLOAD_RATE_LIM_THRESHOLD).await?);
1372 }
1373 }
1374
1375 let mut inner = self.inner.lock().expect("poisoned lock");
1376 let inner = &mut *inner;
1377
1378 let _ = inner.last_uploaded.insert(now);
1379
1380 for period_ctx in inner.time_periods.iter_mut() {
1381 let upload_task_complete_tx = self.upload_task_complete_tx.clone();
1382
1383 // Figure out which HsDirs we need to upload the descriptor to (some of them might already
1384 // have our latest descriptor, so we filter them out).
1385 let hs_dirs = period_ctx
1386 .hs_dirs
1387 .iter()
1388 .filter_map(|(relay_id, status)| {
1389 if *status == DescriptorStatus::Dirty {
1390 Some(relay_id.clone())
1391 } else {
1392 None
1393 }
1394 })
1395 .collect::<Vec<_>>();
1396
1397 if hs_dirs.is_empty() {
1398 trace!("the descriptor is clean for all HSDirs. Nothing to do");
1399 return Ok(());
1400 }
1401
1402 let time_period = period_ctx.params.time_period();
1403 // This scope exists because rng is not Send, so it needs to fall out of scope before we
1404 // await anything.
1405 let netdir = Arc::clone(
1406 inner
1407 .netdir
1408 .as_ref()
1409 .ok_or_else(|| internal!("started upload task without a netdir"))?,
1410 );
1411
1412 let imm = Arc::clone(&self.imm);
1413 let ipt_upload_view = self.ipt_watcher.upload_view();
1414 let config = Arc::clone(&inner.config);
1415 let authorized_clients = authorized_clients.clone();
1416
1417 trace!(nickname=%self.imm.nickname, time_period=?time_period,
1418 "spawning upload task"
1419 );
1420
1421 let params = period_ctx.params.clone();
1422 let shutdown_rx = self.shutdown_tx.subscribe();
1423
1424 // Spawn a task to upload the descriptor to all HsDirs of this time period.
1425 //
1426 // This task will shut down when the reactor is dropped (i.e. when shutdown_rx is
1427 // dropped).
1428 let _handle: () = self
1429 .imm
1430 .runtime
1431 .spawn(async move {
1432 if let Err(e) = Self::upload_for_time_period(
1433 hs_dirs,
1434 &netdir,
1435 config,
1436 params,
1437 Arc::clone(&imm),
1438 ipt_upload_view.clone(),
1439 authorized_clients.clone(),
1440 upload_task_complete_tx,
1441 shutdown_rx,
1442 )
1443 .await
1444 {
1445 error_report!(
1446 e,
1447 "descriptor upload failed for HS service {} and time period {:?}",
1448 imm.nickname,
1449 time_period
1450 );
1451 }
1452 })
1453 .map_err(|e| FatalError::from_spawn("upload_for_time_period task", e))?;
1454 }
1455
1456 Ok(())
1457 }
1458
1459 /// Upload the descriptor for the time period specified in `params`.
1460 ///
1461 /// Failed uploads are retried
1462 /// (see [`upload_descriptor_with_retries`](Reactor::upload_descriptor_with_retries)).
1463 #[allow(clippy::too_many_arguments)] // TODO: refactor
1464 async fn upload_for_time_period(
1465 hs_dirs: Vec<RelayIds>,
1466 netdir: &Arc<NetDir>,
1467 config: Arc<OnionServiceConfigPublisherView>,
1468 params: HsDirParams,
1469 imm: Arc<Immutable<R, M>>,
1470 ipt_upload_view: IptsPublisherUploadView,
1471 authorized_clients: Option<Arc<RestrictedDiscoveryKeys>>,
1472 mut upload_task_complete_tx: mpsc::Sender<TimePeriodUploadResult>,
1473 shutdown_rx: broadcast::Receiver<Void>,
1474 ) -> Result<(), FatalError> {
1475 let time_period = params.time_period();
1476 trace!(time_period=?time_period, "uploading descriptor to all HSDirs for this time period");
1477
1478 let hsdir_count = hs_dirs.len();
1479
1480 /// An error returned from an upload future.
1481 //
1482 // Exhaustive, because this is a private type.
1483 #[derive(Clone, Debug, thiserror::Error)]
1484 enum PublishError {
1485 /// The upload was aborted because there are no IPTs.
1486 ///
1487 /// This happens because of an inevitable TOCTOU race, where after being notified by
1488 /// the IPT manager that the IPTs have changed (via `self.ipt_watcher.await_update`),
1489 /// we find out there actually are no IPTs, so we can't build the descriptor.
1490 ///
1491 /// This is a special kind of error that interrupts the current upload task, and is
1492 /// logged at `debug!` level rather than `warn!` or `error!`.
1493 ///
1494 /// Ideally, this shouldn't happen very often (if at all).
1495 #[error("No IPTs")]
1496 NoIpts,
1497
1498 /// The reactor has shut down
1499 #[error("The reactor has shut down")]
1500 Shutdown,
1501
1502 /// An fatal error.
1503 #[error("{0}")]
1504 Fatal(#[from] FatalError),
1505 }
1506
1507 let max_hsdesc_len: usize = netdir
1508 .params()
1509 .hsdir_max_desc_size
1510 .try_into()
1511 .expect("Unable to convert positive int32 to usize!?");
1512
1513 let upload_results = futures::stream::iter(hs_dirs)
1514 .map(|relay_ids| {
1515 let netdir = netdir.clone();
1516 let config = Arc::clone(&config);
1517 let imm = Arc::clone(&imm);
1518 let ipt_upload_view = ipt_upload_view.clone();
1519 let authorized_clients = authorized_clients.clone();
1520 let params = params.clone();
1521 let mut shutdown_rx = shutdown_rx.clone();
1522
1523 let ed_id = relay_ids
1524 .rsa_identity()
1525 .map(|id| id.to_string())
1526 .unwrap_or_else(|| "unknown".into());
1527 let rsa_id = relay_ids
1528 .rsa_identity()
1529 .map(|id| id.to_string())
1530 .unwrap_or_else(|| "unknown".into());
1531
1532 async move {
1533 let run_upload = |desc| async {
1534 let Some(hsdir) = netdir.by_ids(&relay_ids) else {
1535 // This should never happen (all of our relay_ids are from the stored
1536 // netdir).
1537 let err =
1538 "tried to upload descriptor to relay not found in consensus?!";
1539 warn!(
1540 nickname=%imm.nickname, hsdir_id=%ed_id, hsdir_rsa_id=%rsa_id,
1541 "{err}"
1542 );
1543 return Err(internal!("{err}").into());
1544 };
1545
1546 Self::upload_descriptor_with_retries(
1547 desc,
1548 &netdir,
1549 &hsdir,
1550 &ed_id,
1551 &rsa_id,
1552 Arc::clone(&imm),
1553 )
1554 .await
1555 };
1556
1557 // How long until we're supposed to time out?
1558 let worst_case_end = imm.runtime.now() + OVERALL_UPLOAD_TIMEOUT;
1559 // We generate a new descriptor before _each_ HsDir upload. This means each
1560 // HsDir could, in theory, receive a different descriptor (not just in terms of
1561 // revision-counters, but also with a different set of IPTs). It may seem like
1562 // this could lead to some HsDirs being left with an outdated descriptor, but
1563 // that's not the case: after the upload completes, the publisher will be
1564 // notified by the ipt_watcher of the IPT change event (if there was one to
1565 // begin with), which will trigger another upload job.
1566 let hsdesc = {
1567 // This scope is needed because the ipt_set MutexGuard is not Send, so it
1568 // needs to fall out of scope before the await point below
1569 let mut ipt_set = ipt_upload_view.borrow_for_publish();
1570
1571 // If there are no IPTs, we abort the upload. At this point, we might have
1572 // uploaded the descriptor to some, but not all, HSDirs from the specified
1573 // time period.
1574 //
1575 // Returning an error here means the upload completion task is never
1576 // notified of the outcome of any of these uploads (which means the
1577 // descriptor is not marked clean). This is OK, because if we suddenly find
1578 // out we have no IPTs, it means our built `hsdesc` has an outdated set of
1579 // IPTs, so we need to go back to the main loop to wait for IPT changes,
1580 // and generate a fresh descriptor anyway.
1581 //
1582 // Ideally, this shouldn't happen very often (if at all).
1583 let Some(ipts) = ipt_set.ipts.as_mut() else {
1584 return Err(PublishError::NoIpts);
1585 };
1586
1587 let hsdesc = {
1588 trace!(
1589 nickname=%imm.nickname, time_period=?time_period,
1590 "building descriptor"
1591 );
1592 let mut rng = imm.mockable.thread_rng();
1593 let mut key_rng = tor_llcrypto::rng::CautiousRng;
1594
1595 // We're about to generate a new version of the descriptor,
1596 // so let's generate a new revision counter.
1597 let now = imm.runtime.wallclock();
1598 let revision_counter = imm.generate_revision_counter(¶ms, now)?;
1599
1600 build_sign(
1601 &imm.keymgr,
1602 &imm.pow_manager,
1603 &config,
1604 netdir.params(),
1605 authorized_clients.as_deref(),
1606 ipts,
1607 time_period,
1608 revision_counter,
1609 &mut rng,
1610 &mut key_rng,
1611 imm.runtime.wallclock(),
1612 max_hsdesc_len,
1613 )?
1614 };
1615
1616 if let Err(e) =
1617 ipt_set.note_publication_attempt(&imm.runtime, worst_case_end)
1618 {
1619 let wait = e.log_retry_max(&imm.nickname)?;
1620 // TODO (#1226): retry instead of this
1621 return Err(FatalError::Bug(internal!(
1622 "ought to retry after {wait:?}, crashing instead"
1623 ))
1624 .into());
1625 }
1626
1627 hsdesc
1628 };
1629
1630 let VersionedDescriptor {
1631 desc,
1632 revision_counter,
1633 } = hsdesc;
1634 let desc: Arc<str> = desc.into();
1635
1636 trace!(
1637 nickname=%imm.nickname, time_period=?time_period,
1638 revision_counter=?revision_counter,
1639 "generated new descriptor for time period",
1640 );
1641
1642 // (Actually launch the upload attempt. No timeout is needed
1643 // here, since the backoff::Runner code will handle that for us.)
1644 let upload_res: UploadResult = select_biased! {
1645 shutdown = shutdown_rx.next().fuse() => {
1646 // This will always be None, since Void is uninhabited.
1647 let _: Option<Void> = shutdown;
1648
1649 // It looks like the reactor has shut down,
1650 // so there is no point in uploading the descriptor anymore.
1651 //
1652 // Let's shut down the upload task too.
1653 trace!(
1654 nickname=%imm.nickname, time_period=?time_period,
1655 "upload task received shutdown signal"
1656 );
1657
1658 return Err(PublishError::Shutdown);
1659 },
1660 res = run_upload(desc.clone()).fuse() => res,
1661 };
1662
1663 // Note: UploadResult::Failure is only returned when
1664 // upload_descriptor_with_retries fails, i.e. if all our retry
1665 // attempts have failed
1666 Ok(HsDirUploadStatus {
1667 relay_ids,
1668 upload_res,
1669 revision_counter,
1670 })
1671 }
1672 })
1673 // This fails to compile unless the stream is boxed. See https://github.com/rust-lang/rust/issues/104382
1674 .boxed()
1675 .buffer_unordered(MAX_CONCURRENT_UPLOADS)
1676 .try_collect::<Vec<_>>()
1677 .await;
1678
1679 let upload_results = match upload_results {
1680 Ok(v) => v,
1681 Err(PublishError::Fatal(e)) => return Err(e),
1682 Err(PublishError::NoIpts) => {
1683 debug!(
1684 nickname=%imm.nickname, time_period=?time_period,
1685 "no introduction points; skipping upload"
1686 );
1687
1688 return Ok(());
1689 }
1690 Err(PublishError::Shutdown) => {
1691 debug!(
1692 nickname=%imm.nickname, time_period=?time_period,
1693 "the reactor has shut down; aborting upload"
1694 );
1695
1696 return Ok(());
1697 }
1698 };
1699
1700 let (succeeded, _failed): (Vec<_>, Vec<_>) = upload_results
1701 .iter()
1702 .partition(|res| res.upload_res.is_ok());
1703
1704 debug!(
1705 nickname=%imm.nickname, time_period=?time_period,
1706 "descriptor uploaded successfully to {}/{} HSDirs",
1707 succeeded.len(), hsdir_count
1708 );
1709
1710 if upload_task_complete_tx
1711 .send(TimePeriodUploadResult {
1712 time_period,
1713 hsdir_result: upload_results,
1714 })
1715 .await
1716 .is_err()
1717 {
1718 return Err(internal!(
1719 "failed to notify reactor of upload completion (reactor shut down)"
1720 )
1721 .into());
1722 }
1723
1724 Ok(())
1725 }
1726
1727 /// Upload a descriptor to the specified HSDir.
1728 ///
1729 /// If an upload fails, this returns an `Err`. This function does not handle retries. It is up
1730 /// to the caller to retry on failure.
1731 ///
1732 /// This function does not handle timeouts.
1733 async fn upload_descriptor(
1734 hsdesc: Arc<str>,
1735 netdir: &Arc<NetDir>,
1736 hsdir: &Relay<'_>,
1737 imm: Arc<Immutable<R, M>>,
1738 ) -> Result<(), UploadError> {
1739 let request = HsDescUploadRequest::new(hsdesc);
1740
1741 trace!(nickname=%imm.nickname, hsdir_id=%hsdir.id(), hsdir_rsa_id=%hsdir.rsa_id(),
1742 "starting descriptor upload",
1743 );
1744
1745 let tunnel = imm
1746 .mockable
1747 .get_or_launch_hs_dir(netdir, OwnedCircTarget::from_circ_target(hsdir))
1748 .await?;
1749 let source: Option<SourceInfo> = tunnel
1750 .source_info()
1751 .map_err(into_internal!("Couldn't get SourceInfo for circuit"))?;
1752
1753 let mut stream = tunnel
1754 .begin_dir_stream()
1755 .await
1756 .map_err(UploadError::Stream)?;
1757
1758 let _response: String = send_request(&imm.runtime, &request, &mut stream, source)
1759 .await
1760 .map_err(|dir_error| -> UploadError {
1761 match dir_error {
1762 DirClientError::RequestFailed(e) => e.into(),
1763 DirClientError::CircMgr(e) => into_internal!(
1764 "tor-dirclient complains about circmgr going wrong but we gave it a stream"
1765 )(e)
1766 .into(),
1767 e => into_internal!("unexpected error")(e).into(),
1768 }
1769 })?
1770 .into_output_string()?; // This returns an error if we received an error response
1771
1772 Ok(())
1773 }
1774
1775 /// Upload a descriptor to the specified HSDir, retrying if appropriate.
1776 ///
1777 /// Any failed uploads are retried according to a [`PublisherBackoffSchedule`].
1778 /// Each failed upload is retried until it succeeds, or until the overall timeout specified
1779 /// by [`BackoffSchedule::overall_timeout`] elapses. Individual attempts are timed out
1780 /// according to the [`BackoffSchedule::single_attempt_timeout`].
1781 /// This function gives up after the overall timeout elapses,
1782 /// declaring the upload a failure, and never retrying it again.
1783 ///
1784 /// See also [`BackoffSchedule`].
1785 async fn upload_descriptor_with_retries(
1786 hsdesc: Arc<str>,
1787 netdir: &Arc<NetDir>,
1788 hsdir: &Relay<'_>,
1789 ed_id: &str,
1790 rsa_id: &str,
1791 imm: Arc<Immutable<R, M>>,
1792 ) -> UploadResult {
1793 /// The base delay to use for the backoff schedule.
1794 const BASE_DELAY_MSEC: u32 = 1000;
1795 let schedule = PublisherBackoffSchedule {
1796 retry_delay: RetryDelay::from_msec(BASE_DELAY_MSEC),
1797 mockable: imm.mockable.clone(),
1798 };
1799
1800 let runner = Runner::new(
1801 "upload a hidden service descriptor".into(),
1802 schedule.clone(),
1803 imm.runtime.clone(),
1804 );
1805
1806 let fallible_op = || async {
1807 let r = Self::upload_descriptor(hsdesc.clone(), netdir, hsdir, Arc::clone(&imm)).await;
1808
1809 if let Err(e) = &r {
1810 if e.should_report_as_suspicious() {
1811 // Note that not every protocol violation is suspicious:
1812 // we only warn on the protocol violations that look like attempts
1813 // to do a traffic tagging attack via hsdir inflation.
1814 // (See proposal 360.)
1815 warn_report!(
1816 e,
1817 "Suspicious error while uploading descriptor to {}/{}",
1818 ed_id,
1819 rsa_id
1820 );
1821 }
1822
1823 if let UploadError::Request(req_err) = e {
1824 if let RequestError::HttpStatus(status, _msg) = &req_err.error {
1825 if (400..500).contains(status) {
1826 // This should never happen (it's a bug if it does!)
1827 log_ratelim!(
1828 "The peer says our HS descriptor upload request was invalid. This is a bug (maybe theirs; maybe ours)";
1829 Err::<(), _>(e.clone());
1830 );
1831 }
1832
1833 // C Tor HsDirs return a 400 status code if they deem
1834 // the descriptor to be invalid.
1835 if *status == 400 {
1836 let res = dump_invalid_hsdesc(&hsdesc, &imm);
1837
1838 log_ratelim!(
1839 "Dumping rejected HS descriptor to {}",
1840 imm.bad_hsdescs_dir.as_path().display_lossy();
1841 res;
1842 );
1843
1844 // TODO: we might want to garbage-collect this at some point
1845 // (e.g. if the mtime of the last dumped
1846 // bad descriptor is far enough into the past?)
1847 }
1848 }
1849 }
1850 }
1851 r
1852 };
1853
1854 let outcome: Result<(), BackoffError<UploadError>> = runner.run(fallible_op).await;
1855 match outcome {
1856 Ok(()) => {
1857 debug!(
1858 nickname=%imm.nickname, hsdir_id=%ed_id, hsdir_rsa_id=%rsa_id,
1859 "successfully uploaded descriptor to HSDir",
1860 );
1861
1862 Ok(())
1863 }
1864 Err(e) => {
1865 warn_report!(
1866 e,
1867 "failed to upload descriptor for service {} (hsdir_id={}, hsdir_rsa_id={})",
1868 imm.nickname,
1869 ed_id,
1870 rsa_id
1871 );
1872
1873 Err(e.into())
1874 }
1875 }
1876 }
1877
1878 /// Stop publishing descriptors until the specified delay elapses.
1879 async fn start_rate_limit(&mut self, delay: Duration) -> Result<(), Bug> {
1880 if !matches!(self.status(), PublishStatus::RateLimited(_)) {
1881 debug!(
1882 "We are rate-limited for {}; pausing descriptor publication",
1883 humantime::format_duration(delay)
1884 );
1885 let until = self.imm.runtime.now() + delay;
1886 self.update_publish_status(PublishStatus::RateLimited(until))
1887 .await?;
1888 }
1889
1890 Ok(())
1891 }
1892
1893 /// Handle the upload rate-limit being lifted.
1894 async fn expire_rate_limit(&mut self) -> Result<(), Bug> {
1895 debug!("We are no longer rate-limited; resuming descriptor publication");
1896 self.update_publish_status(PublishStatus::UploadScheduled)
1897 .await?;
1898 Ok(())
1899 }
1900
1901 /// Return the authorized clients, if restricted mode is enabled.
1902 ///
1903 /// Returns `Ok(None)` if restricted discovery mode is disabled.
1904 ///
1905 /// Returns an error if restricted discovery mode is enabled, but the client list is empty.
1906 #[cfg_attr(
1907 not(feature = "restricted-discovery"),
1908 allow(clippy::unnecessary_wraps)
1909 )]
1910 fn authorized_clients(&self) -> Result<Option<Arc<RestrictedDiscoveryKeys>>, FatalError> {
1911 cfg_if::cfg_if! {
1912 if #[cfg(feature = "restricted-discovery")] {
1913 let authorized_clients = self
1914 .inner
1915 .lock()
1916 .expect("poisoned lock")
1917 .authorized_clients
1918 .clone();
1919
1920 if authorized_clients.as_ref().as_ref().map(|v| v.is_empty()).unwrap_or_default() {
1921 return Err(FatalError::RestrictedDiscoveryNoClients);
1922 }
1923
1924 Ok(authorized_clients)
1925 } else {
1926 Ok(None)
1927 }
1928 }
1929 }
1930}
1931
1932/// Dump an invalid HS descriptor to disk, for debugging purposes.
1933///
1934/// The descriptor will get written to the state directory,
1935/// in `hss/<HS_NICKNAME>/bad_hsdescs/current`.
1936///
1937// TODO(#2760): this is a quick and dirty hack for debugging #2760.
1938// We may want to eventually replace this with a more principled
1939// "invalid descriptor" dumping utility, like the one C Tor has
1940// in dirparse/unparseable.c.
1941fn dump_invalid_hsdesc<R, M>(
1942 hsdesc: &Arc<str>,
1943 imm: &Arc<Immutable<R, M>>,
1944) -> Result<(), HsDescDumpError>
1945where
1946 R: Runtime,
1947 M: Mockable,
1948{
1949 use rand::distr::{Alphanumeric, SampleString};
1950 // If multiple upload tasks have an invalid
1951 // descriptor, they will all race to write to "filename.tmp",
1952 // so we need a unique id for the temporary file
1953 // (TODO: maybe it would be useful to have a FileAccess API for this?):
1954 //
1955 // Case-insensitive file systems are going to have a higher risk
1956 // of tmpname collisions for the tasks trying to write an invalid descriptor,
1957 // but we can try mitigate that by generating a relatively long string
1958 // (in practice, the chances of this happening are going to be very slim).
1959 let mut rng = imm.mockable.thread_rng();
1960 let tmpname = Alphanumeric.sample_string(&mut rng, 16);
1961 imm.bad_hsdescs_dir
1962 .write_and_replace(&tmpname, hsdesc.as_ref())?;
1963
1964 let tmp_path = imm.bad_hsdescs_dir.as_path().join(tmpname);
1965 let target = imm.bad_hsdescs_dir.as_path().join("current");
1966
1967 // This rename() bypasses the fs-mistrust checks, but I believe that's okay,
1968 // because the write_and_replace() above already checked the permissions
1969 // of the parent dir.
1970 let res = std::fs::rename(&tmp_path, &target).map_err(|err| HsDescDumpError::Rename {
1971 filename: target,
1972 err: Arc::new(err),
1973 });
1974
1975 if res.is_err() {
1976 // Try to at least clean up the temp file...
1977 let _ = std::fs::remove_file(tmp_path);
1978 }
1979
1980 Ok(())
1981}
1982
1983/// An error returned when trying to dump an invalid HS desc to disk.
1984#[derive(Clone, Debug, thiserror::Error)]
1985enum HsDescDumpError {
1986 /// We encountered a problem while inspecting or creating a directory.
1987 #[error("Problem accessing filesystem")]
1988 FsMistrust(#[from] fs_mistrust::Error),
1989
1990 /// We encountered an error while attempting an IO operation on a file.
1991 #[error(r#"IO error while attempting to rename file to {}"#, filename.anonymize_home())]
1992 Rename {
1993 /// The file that we were trying to modify or inspect
1994 filename: PathBuf,
1995 /// The error that we got when trying to perform the operation.
1996 #[source]
1997 err: Arc<std::io::Error>,
1998 },
1999}
2000
2001/// Try to expand a path, logging a warning on failure.
2002fn maybe_expand_path(p: &CfgPath, r: &CfgPathResolver) -> Option<PathBuf> {
2003 // map_err returns unit for clarity
2004 #[allow(clippy::unused_unit, clippy::semicolon_if_nothing_returned)]
2005 p.path(r)
2006 .map_err(|e| {
2007 tor_error::warn_report!(e, "invalid path");
2008 ()
2009 })
2010 .ok()
2011}
2012
2013/// Add `path` to the specified `watcher`.
2014macro_rules! watch_path {
2015 ($watcher:expr, $path:expr, $watch_fn:ident, $($watch_fn_args:expr,)*) => {{
2016 if let Err(e) = $watcher.$watch_fn(&$path, $($watch_fn_args)*) {
2017 warn_report!(e, "failed to watch path {:?}", $path);
2018 } else {
2019 debug!("watching path {:?}", $path);
2020 }
2021 }}
2022}
2023
2024/// Add the specified directories to the watcher.
2025fn watch_dirs<R: Runtime>(
2026 watcher: &mut FileWatcherBuilder<R>,
2027 dirs: &DirectoryKeyProviderList,
2028 path_resolver: &CfgPathResolver,
2029) {
2030 for path in dirs {
2031 let path = path.path();
2032 let Some(path) = maybe_expand_path(path, path_resolver) else {
2033 warn!("failed to expand key_dir path {:?}", path);
2034 continue;
2035 };
2036
2037 // If the path doesn't exist, the notify watcher will return an error if we attempt to watch it,
2038 // so we skip over paths that don't exist at this time
2039 // (this obviously suffers from a TOCTOU race, but most of the time,
2040 // it is good enough at preventing the watcher from failing to watch.
2041 // If the race *does* happen it is not disastrous, i.e. the reactor won't crash,
2042 // but it will fail to set the watcher).
2043 if matches!(path.try_exists(), Ok(true)) {
2044 watch_path!(watcher, &path, watch_dir, "auth",);
2045 }
2046 // FileWatcher::watch_path causes the parent dir of the path to be watched.
2047 if matches!(path.parent().map(|p| p.try_exists()), Some(Ok(true))) {
2048 watch_path!(watcher, &path, watch_path,);
2049 }
2050 }
2051}
2052
2053/// Try to read the blinded identity key for a given `TimePeriod`.
2054///
2055/// Returns `None` if the service is running in "offline" mode.
2056///
2057// TODO (#1194): we don't currently have support for "offline" mode so this can never return
2058// `Ok(None)`.
2059pub(super) fn read_blind_id_keypair(
2060 keymgr: &Arc<KeyMgr>,
2061 nickname: &HsNickname,
2062 period: TimePeriod,
2063) -> Result<Option<HsBlindIdKeypair>, FatalError> {
2064 let svc_key_spec = HsIdKeypairSpecifier::new(nickname.clone());
2065 let hsid_kp = keymgr
2066 .get::<HsIdKeypair>(&svc_key_spec)?
2067 .ok_or_else(|| FatalError::MissingHsIdKeypair(nickname.clone()))?;
2068
2069 let blind_id_key_spec = BlindIdKeypairSpecifier::new(nickname.clone(), period);
2070
2071 // TODO: make the keystore selector configurable
2072 let keystore_selector = Default::default();
2073 match keymgr.get::<HsBlindIdKeypair>(&blind_id_key_spec)? {
2074 Some(kp) => Ok(Some(kp)),
2075 None => {
2076 let (_hs_blind_id_key, hs_blind_id_kp, _subcredential) = hsid_kp
2077 .compute_blinded_key(period)
2078 .map_err(|_| internal!("failed to compute blinded key"))?;
2079
2080 // Note: we can't use KeyMgr::generate because this key is derived from the HsId
2081 // (KeyMgr::generate uses the tor_keymgr::Keygen trait under the hood,
2082 // which assumes keys are randomly generated, rather than derived from existing keys).
2083
2084 keymgr.insert(hs_blind_id_kp, &blind_id_key_spec, keystore_selector, true)?;
2085
2086 let arti_path = |spec: &dyn KeySpecifier| {
2087 spec.arti_path()
2088 .map_err(into_internal!("invalid key specifier?!"))
2089 };
2090
2091 Ok(Some(
2092 keymgr.get::<HsBlindIdKeypair>(&blind_id_key_spec)?.ok_or(
2093 FatalError::KeystoreRace {
2094 action: "read",
2095 path: arti_path(&blind_id_key_spec)?,
2096 },
2097 )?,
2098 ))
2099 }
2100 }
2101}
2102
2103/// Determine the [`State`] of the publisher based on the upload results
2104/// from the current `time_periods`.
2105fn upload_result_state(
2106 netdir: &NetDir,
2107 time_periods: &[TimePeriodContext],
2108) -> (State, Option<Problem>) {
2109 let current_period = netdir.hs_time_period();
2110 let current_period_res = time_periods
2111 .iter()
2112 .find(|ctx| ctx.params.time_period() == current_period);
2113
2114 let succeeded_current_tp = current_period_res
2115 .iter()
2116 .flat_map(|res| &res.upload_results)
2117 .filter(|res| res.upload_res.is_ok())
2118 .collect_vec();
2119
2120 let secondary_tp_res = time_periods
2121 .iter()
2122 .filter(|ctx| ctx.params.time_period() != current_period)
2123 .collect_vec();
2124
2125 let succeeded_secondary_tp = secondary_tp_res
2126 .iter()
2127 .flat_map(|res| &res.upload_results)
2128 .filter(|res| res.upload_res.is_ok())
2129 .collect_vec();
2130
2131 // All of the failed uploads (for all TPs)
2132 let failed = time_periods
2133 .iter()
2134 .flat_map(|res| &res.upload_results)
2135 .filter(|res| res.upload_res.is_err())
2136 .collect_vec();
2137 let problems: Vec<DescUploadRetryError> = failed
2138 .iter()
2139 .flat_map(|e| e.upload_res.as_ref().map_err(|e| e.clone()).err())
2140 .collect();
2141
2142 let err = match problems.as_slice() {
2143 [_, ..] => Some(problems.into()),
2144 [] => None,
2145 };
2146
2147 if time_periods.len() < 2 {
2148 // We need at least TP contexts (one for the primary TP,
2149 // and another for the secondary one).
2150 //
2151 // If either is missing, we are unreachable for some or all clients.
2152 return (State::DegradedUnreachable, err);
2153 }
2154
2155 let state = match (
2156 succeeded_current_tp.as_slice(),
2157 succeeded_secondary_tp.as_slice(),
2158 ) {
2159 (&[], &[..]) | (&[..], &[]) if failed.is_empty() => {
2160 // We don't have any upload results for one or both TPs.
2161 // We are still bootstrapping.
2162 State::Bootstrapping
2163 }
2164 (&[_, ..], &[_, ..]) if failed.is_empty() => {
2165 // We have uploaded the descriptor to one or more HsDirs from both
2166 // HsDir rings (primary and secondary), and none of the uploads failed.
2167 // We are fully reachable.
2168 State::Running
2169 }
2170 (&[_, ..], &[_, ..]) => {
2171 // We have uploaded the descriptor to one or more HsDirs from both
2172 // HsDir rings (primary and secondary), but some of the uploads failed.
2173 // We are reachable, but we failed to upload the descriptor to all the HsDirs
2174 // that were supposed to have it.
2175 State::DegradedReachable
2176 }
2177 (&[..], &[]) | (&[], &[..]) => {
2178 // We have either
2179 // * uploaded the descriptor to some of the HsDirs from one of the rings,
2180 // but haven't managed to upload it to any of the HsDirs on the other ring, or
2181 // * all of the uploads failed
2182 //
2183 // Either way, we are definitely not reachable by all clients.
2184 State::DegradedUnreachable
2185 }
2186 };
2187
2188 (state, err)
2189}
2190
2191/// Whether the reactor should initiate an upload.
2192#[derive(Copy, Clone, Debug, Default, PartialEq)]
2193enum PublishStatus {
2194 /// We need to call upload_all.
2195 UploadScheduled,
2196 /// We are rate-limited until the specified [`Instant`].
2197 ///
2198 /// We have tried to schedule multiple uploads in a short time span,
2199 /// and we are rate-limited. We are waiting for a signal from the schedule_upload_tx
2200 /// channel to unblock us.
2201 RateLimited(Instant),
2202 /// We are idle and waiting for external events.
2203 ///
2204 /// We have enough information to build the descriptor, but since we have already called
2205 /// upload_all to upload it to all relevant HSDirs, there is nothing for us to do right nbow.
2206 Idle,
2207 /// We are waiting for the IPT manager to establish some introduction points.
2208 ///
2209 /// No descriptors will be published until the `PublishStatus` of the reactor is changed to
2210 /// `UploadScheduled`.
2211 #[default]
2212 AwaitingIpts,
2213}
2214
2215/// The backoff schedule for the task that publishes descriptors.
2216#[derive(Clone, Debug)]
2217struct PublisherBackoffSchedule<M: Mockable> {
2218 /// The delays
2219 retry_delay: RetryDelay,
2220 /// The mockable reactor state, needed for obtaining an rng.
2221 mockable: M,
2222}
2223
2224impl<M: Mockable> BackoffSchedule for PublisherBackoffSchedule<M> {
2225 fn max_retries(&self) -> Option<usize> {
2226 None
2227 }
2228
2229 fn overall_timeout(&self) -> Option<Duration> {
2230 Some(OVERALL_UPLOAD_TIMEOUT)
2231 }
2232
2233 fn single_attempt_timeout(&self) -> Option<Duration> {
2234 Some(self.mockable.estimate_upload_timeout())
2235 }
2236
2237 fn next_delay<E: RetriableError>(&mut self, _error: &E) -> Option<Duration> {
2238 Some(self.retry_delay.next_delay(&mut self.mockable.thread_rng()))
2239 }
2240}
2241
2242impl RetriableError for UploadError {
2243 fn should_retry(&self) -> bool {
2244 match self {
2245 UploadError::Request(_) | UploadError::Circuit(_) | UploadError::Stream(_) => true,
2246 UploadError::Bug(_) => false,
2247 }
2248 }
2249}
2250
2251/// The outcome of uploading a descriptor to the HSDirs from a particular time period.
2252#[derive(Debug, Clone)]
2253struct TimePeriodUploadResult {
2254 /// The time period.
2255 time_period: TimePeriod,
2256 /// The upload results.
2257 hsdir_result: Vec<HsDirUploadStatus>,
2258}
2259
2260/// The outcome of uploading a descriptor to a particular HsDir.
2261#[derive(Clone, Debug)]
2262struct HsDirUploadStatus {
2263 /// The identity of the HsDir we attempted to upload the descriptor to.
2264 relay_ids: RelayIds,
2265 /// The outcome of this attempt.
2266 upload_res: UploadResult,
2267 /// The revision counter of the descriptor we tried to upload.
2268 revision_counter: RevisionCounter,
2269}
2270
2271/// The outcome of uploading a descriptor.
2272type UploadResult = Result<(), DescUploadRetryError>;
2273
2274impl From<BackoffError<UploadError>> for DescUploadRetryError {
2275 fn from(e: BackoffError<UploadError>) -> Self {
2276 use BackoffError as BE;
2277 use DescUploadRetryError as DURE;
2278
2279 match e {
2280 BE::FatalError(e) => DURE::FatalError(e),
2281 BE::MaxRetryCountExceeded(e) => DURE::MaxRetryCountExceeded(e),
2282 BE::Timeout(e) => DURE::Timeout(e),
2283 BE::ExplicitStop(_) => {
2284 DURE::Bug(internal!("explicit stop in publisher backoff schedule?!"))
2285 }
2286 }
2287 }
2288}
2289
2290// NOTE: the rest of the publisher tests live in publish.rs
2291#[cfg(test)]
2292mod test {
2293 // @@ begin test lint list maintained by maint/add_warning @@
2294 #![allow(clippy::bool_assert_comparison)]
2295 #![allow(clippy::clone_on_copy)]
2296 #![allow(clippy::dbg_macro)]
2297 #![allow(clippy::mixed_attributes_style)]
2298 #![allow(clippy::print_stderr)]
2299 #![allow(clippy::print_stdout)]
2300 #![allow(clippy::single_char_pattern)]
2301 #![allow(clippy::unwrap_used)]
2302 #![allow(clippy::unchecked_time_subtraction)]
2303 #![allow(clippy::useless_vec)]
2304 #![allow(clippy::needless_pass_by_value)]
2305 #![allow(clippy::string_slice)] // See arti#2571
2306 //! <!-- @@ end test lint list maintained by maint/add_warning @@ -->
2307 use super::*;
2308 use tor_netdir::testnet;
2309
2310 /// Create a `TimePeriodContext` from the specified upload results.
2311 fn create_time_period_ctx(
2312 params: &HsDirParams,
2313 upload_results: Vec<HsDirUploadStatus>,
2314 ) -> TimePeriodContext {
2315 TimePeriodContext {
2316 params: params.clone(),
2317 hs_dirs: vec![],
2318 last_successful: None,
2319 upload_results,
2320 }
2321 }
2322
2323 /// Create a single `HsDirUploadStatus`
2324 fn create_upload_status(upload_res: UploadResult) -> HsDirUploadStatus {
2325 HsDirUploadStatus {
2326 relay_ids: RelayIds::empty(),
2327 upload_res,
2328 revision_counter: RevisionCounter::from(13),
2329 }
2330 }
2331
2332 /// Create a bunch of results, all with the specified `upload_res`.
2333 fn create_upload_results(upload_res: UploadResult) -> Vec<HsDirUploadStatus> {
2334 std::iter::repeat_with(|| create_upload_status(upload_res.clone()))
2335 .take(10)
2336 .collect()
2337 }
2338
2339 fn construct_netdir() -> NetDir {
2340 const SRV1: [u8; 32] = *b"The door refused to open. ";
2341 const SRV2: [u8; 32] = *b"It said, 'Five cents, please.' ";
2342
2343 let dir = testnet::construct_custom_netdir(|_, _, bld| {
2344 bld.shared_rand_cur(7, SRV1.into(), None)
2345 .shared_rand_prev(7, SRV2.into(), None);
2346 })
2347 .unwrap();
2348
2349 dir.unwrap_if_sufficient().unwrap()
2350 }
2351
2352 #[test]
2353 fn upload_result_status_bootstrapping() {
2354 let netdir = construct_netdir();
2355 let all_params = netdir.hs_all_time_periods();
2356 let current_period = netdir.hs_time_period();
2357 let primary_params = all_params
2358 .iter()
2359 .find(|param| param.time_period() == current_period)
2360 .unwrap();
2361 let results = [
2362 (vec![], vec![]),
2363 (vec![], create_upload_results(Ok(()))),
2364 (create_upload_results(Ok(())), vec![]),
2365 ];
2366
2367 for (primary_result, secondary_result) in results {
2368 let primary_ctx = create_time_period_ctx(primary_params, primary_result);
2369
2370 let secondary_params = all_params
2371 .iter()
2372 .find(|param| param.time_period() != current_period)
2373 .unwrap();
2374 let secondary_ctx = create_time_period_ctx(secondary_params, secondary_result.clone());
2375
2376 let (status, err) = upload_result_state(&netdir, &[primary_ctx, secondary_ctx]);
2377 assert_eq!(status, State::Bootstrapping);
2378 assert!(err.is_none());
2379 }
2380 }
2381
2382 #[test]
2383 fn upload_result_status_running() {
2384 let netdir = construct_netdir();
2385 let all_params = netdir.hs_all_time_periods();
2386 let current_period = netdir.hs_time_period();
2387 let primary_params = all_params
2388 .iter()
2389 .find(|param| param.time_period() == current_period)
2390 .unwrap();
2391
2392 let secondary_result = create_upload_results(Ok(()));
2393 let secondary_params = all_params
2394 .iter()
2395 .find(|param| param.time_period() != current_period)
2396 .unwrap();
2397 let secondary_ctx = create_time_period_ctx(secondary_params, secondary_result.clone());
2398
2399 let primary_result = create_upload_results(Ok(()));
2400 let primary_ctx = create_time_period_ctx(primary_params, primary_result);
2401 let (status, err) = upload_result_state(&netdir, &[primary_ctx, secondary_ctx]);
2402 assert_eq!(status, State::Running);
2403 assert!(err.is_none());
2404 }
2405
2406 #[test]
2407 fn upload_result_status_reachable() {
2408 let netdir = construct_netdir();
2409 let all_params = netdir.hs_all_time_periods();
2410 let current_period = netdir.hs_time_period();
2411 let primary_params = all_params
2412 .iter()
2413 .find(|param| param.time_period() == current_period)
2414 .unwrap();
2415
2416 let primary_result = create_upload_results(Ok(()));
2417 let primary_ctx = create_time_period_ctx(primary_params, primary_result.clone());
2418 let failed_res = create_upload_results(Err(DescUploadRetryError::Bug(internal!("test"))));
2419 let secondary_result = create_upload_results(Ok(()))
2420 .into_iter()
2421 .chain(failed_res.iter().cloned())
2422 .collect();
2423 let secondary_params = all_params
2424 .iter()
2425 .find(|param| param.time_period() != current_period)
2426 .unwrap();
2427 let secondary_ctx = create_time_period_ctx(secondary_params, secondary_result);
2428 let (status, err) = upload_result_state(&netdir, &[primary_ctx, secondary_ctx]);
2429
2430 // Degraded but reachable (because some of the secondary HsDir uploads failed).
2431 assert_eq!(status, State::DegradedReachable);
2432 assert!(matches!(err, Some(Problem::DescriptorUpload(_))));
2433 }
2434
2435 #[test]
2436 fn upload_result_status_unreachable() {
2437 let netdir = construct_netdir();
2438 let all_params = netdir.hs_all_time_periods();
2439 let current_period = netdir.hs_time_period();
2440 let primary_params = all_params
2441 .iter()
2442 .find(|param| param.time_period() == current_period)
2443 .unwrap();
2444 let mut primary_result =
2445 create_upload_results(Err(DescUploadRetryError::Bug(internal!("test"))));
2446 let primary_ctx = create_time_period_ctx(primary_params, primary_result.clone());
2447 // No secondary TP (we are unreachable).
2448 let (status, err) = upload_result_state(&netdir, &[primary_ctx]);
2449 assert_eq!(status, State::DegradedUnreachable);
2450 assert!(matches!(err, Some(Problem::DescriptorUpload(_))));
2451
2452 // Add a successful result
2453 primary_result.push(create_upload_status(Ok(())));
2454 let primary_ctx = create_time_period_ctx(primary_params, primary_result.clone());
2455 let (status, err) = upload_result_state(&netdir, &[primary_ctx]);
2456 // Still degraded, and unreachable (because we don't have a TimePeriodContext
2457 // for the secondary TP)
2458 assert_eq!(status, State::DegradedUnreachable);
2459 assert!(matches!(err, Some(Problem::DescriptorUpload(_))));
2460
2461 // If we add another time period where none of the uploads were successful,
2462 // we're *still* unreachable
2463 let secondary_result =
2464 create_upload_results(Err(DescUploadRetryError::Bug(internal!("test"))));
2465 let secondary_params = all_params
2466 .iter()
2467 .find(|param| param.time_period() != current_period)
2468 .unwrap();
2469 let secondary_ctx = create_time_period_ctx(secondary_params, secondary_result.clone());
2470 let primary_ctx = create_time_period_ctx(primary_params, primary_result.clone());
2471 let (status, err) = upload_result_state(&netdir, &[primary_ctx, secondary_ctx]);
2472 assert_eq!(status, State::DegradedUnreachable);
2473 assert!(matches!(err, Some(Problem::DescriptorUpload(_))));
2474 }
2475}