Skip to main content

tor_hsservice/publish/
reactor.rs

1//! The onion service publisher reactor.
2//!
3//! Generates and publishes hidden service descriptors in response to various events.
4//!
5//! [`Reactor::run`] is the entry-point of the reactor. It starts the reactor,
6//! and runs until [`Reactor::run_once`] returns [`ShutdownStatus::Terminate`]
7//! or a fatal error occurs. `ShutdownStatus::Terminate` is returned if
8//! any of the channels the reactor is receiving events from is closed
9//! (i.e. when the senders are dropped).
10//!
11//! ## Publisher status
12//!
13//! The publisher has an internal [`PublishStatus`], distinct from its [`State`],
14//! which is used for onion service status reporting.
15//!
16//! The main loop of the reactor reads the current `PublishStatus` from `publish_status_rx`,
17//! and responds by generating and publishing a new descriptor if needed.
18//!
19//! See [`PublishStatus`] and [`Reactor::publish_status_rx`] for more details.
20//!
21//! ## When do we publish?
22//!
23//! We generate and publish a new descriptor if
24//!   * the introduction points have changed
25//!   * the onion service configuration has changed in a meaningful way (for example,
26//!     if the `restricted_discovery` configuration or its [`Anonymity`](crate::Anonymity)
27//!     has changed. See [`OnionServiceConfigPublisherView`]).
28//!   * there is a new consensus
29//!   * it is time to republish the descriptor (after we upload a descriptor,
30//!     we schedule it for republishing at a random time between 60 minutes and 120 minutes
31//!     in the future)
32//!
33//! ## Onion service status
34//!
35//! With respect to [`OnionServiceStatus`] reporting,
36//! the following state transitions are possible:
37//!
38//!
39//! ```ignore
40//!
41//!                 update_publish_status(UploadScheduled|AwaitingIpts|RateLimited)
42//!                +---------------------------------------+
43//!                |                                       |
44//!                |                                       v
45//!                |                               +---------------+
46//!                |                               | Bootstrapping |
47//!                |                               +---------------+
48//!                |                                       |
49//!                |                                       |           uploaded to at least
50//!                |  not enough HsDir uploads succeeded   |        some HsDirs from each ring
51//!                |         +-----------------------------+-----------------------+
52//!                |         |                             |                       |
53//!                |         |              all HsDir uploads succeeded            |
54//!                |         |                             |                       |
55//!                |         v                             v                       v
56//!                |  +---------------------+         +---------+        +---------------------+
57//!                |  | DegradedUnreachable |         | Running |        |  DegradedReachable  |
58//! +----------+   |  +---------------------+         +---------+        +---------------------+
59//! | Shutdown |-- |         |                           |                        |
60//! +----------+   |         |                           |                        |
61//!                |         |                           |                        |
62//!                |         |                           |                        |
63//!                |         +---------------------------+------------------------+
64//!                |                                     |   invalid authorized_clients
65//!                |                                     |      after handling config change
66//!                |                                     |
67//!                |                                     v
68//!                |     run_once() returns an error +--------+
69//!                +-------------------------------->| Broken |
70//!                                                  +--------+
71//! ```
72//!
73//! We can also transition from `Broken`, `DegradedReachable`, or `DegradedUnreachable`
74//! back to `Bootstrapping` (those transitions were omitted for brevity).
75
76use tor_circmgr::ServiceOnionServiceDirTunnel;
77use tor_config::file_watcher::{
78    self, Event as FileEvent, FileEventReceiver, FileEventSender, FileWatcher, FileWatcherBuilder,
79};
80use tor_config_path::{CfgPath, CfgPathResolver};
81use tor_dirclient::{RequestError, SourceInfo};
82use tor_netdir::{DirEvent, NetDir};
83use tor_persist::state_dir::InstanceRawSubdir;
84use tracing::instrument;
85
86use crate::config::OnionServiceConfigPublisherView;
87use crate::config::restricted_discovery::{
88    DirectoryKeyProviderList, RestrictedDiscoveryConfig, RestrictedDiscoveryKeys,
89};
90use crate::status::{DescUploadRetryError, Problem};
91
92use super::*;
93use derive_more::From;
94use fs_mistrust::anon_home::PathExt;
95
96// TODO-CLIENT-AUTH: perhaps we should add a separate CONFIG_CHANGE_REPUBLISH_DEBOUNCE_INTERVAL
97// for rate-limiting the publish jobs triggered by a change in the config?
98//
99// Currently the descriptor publish tasks triggered by changes in the config
100// are rate-limited via the usual rate limiting mechanism
101// (which rate-limits the uploads for 1m).
102//
103// I think this is OK for now, but we might need to rethink this if it becomes problematic
104// (for example, we might want an even longer rate-limit, or to reset any existing rate-limits
105// each time the config is modified).
106
107/// The upload rate-limiting threshold.
108///
109/// Before initiating an upload, the reactor checks if the last upload was at least
110/// `UPLOAD_RATE_LIM_THRESHOLD` seconds ago. If so, it uploads the descriptor to all HsDirs that
111/// need it. If not, it schedules the upload to happen `UPLOAD_RATE_LIM_THRESHOLD` seconds from the
112/// current time.
113//
114// TODO: We may someday need to tune this value; it was chosen more or less arbitrarily.
115const UPLOAD_RATE_LIM_THRESHOLD: Duration = Duration::from_secs(60);
116
117/// The maximum number of concurrent upload tasks per time period.
118//
119// TODO: this value was arbitrarily chosen and may not be optimal.  For now, it
120// will have no effect, since the current number of replicas is far less than
121// this value.
122//
123// The uploads for all TPs happen in parallel.  As a result, the actual limit for the maximum
124// number of concurrent upload tasks is multiplied by a number which depends on the TP parameters
125// (currently 2, which means the concurrency limit will, in fact, be 32).
126//
127// We should try to decouple this value from the TP parameters.
128const MAX_CONCURRENT_UPLOADS: usize = 16;
129
130/// The maximum time allowed for uploading a descriptor to a single HSDir,
131/// across all attempts.
132pub(crate) const OVERALL_UPLOAD_TIMEOUT: Duration = Duration::from_secs(5 * 60);
133
134/// A reactor for the HsDir [`Publisher`]
135///
136/// The entrypoint is [`Reactor::run`].
137#[must_use = "If you don't call run() on the reactor, it won't publish any descriptors."]
138pub(super) struct Reactor<R: Runtime, M: Mockable> {
139    /// The immutable, shared inner state.
140    imm: Arc<Immutable<R, M>>,
141    /// A source for new network directories that we use to determine
142    /// our HsDirs.
143    dir_provider: Arc<dyn NetDirProvider>,
144    /// The mutable inner state,
145    inner: Arc<Mutex<Inner>>,
146    /// A channel for receiving IPT change notifications.
147    ipt_watcher: IptsPublisherView,
148    /// A channel for receiving onion service config change notifications.
149    config_rx: watch::Receiver<Arc<OnionServiceConfig>>,
150    /// A channel for receiving restricted discovery key_dirs change notifications.
151    key_dirs_rx: FileEventReceiver,
152    /// A channel for sending restricted discovery key_dirs change notifications.
153    ///
154    /// A copy of this sender is handed out to every `FileWatcher` created.
155    key_dirs_tx: FileEventSender,
156    /// A channel for receiving updates regarding our [`PublishStatus`].
157    ///
158    /// The main loop of the reactor watches for updates on this channel.
159    ///
160    /// When the [`PublishStatus`] changes to [`UploadScheduled`](PublishStatus::UploadScheduled),
161    /// we can start publishing descriptors.
162    ///
163    /// If the [`PublishStatus`] is [`AwaitingIpts`](PublishStatus::AwaitingIpts), publishing is
164    /// paused until we receive a notification on `ipt_watcher` telling us the IPT manager has
165    /// established some introduction points.
166    publish_status_rx: watch::Receiver<PublishStatus>,
167    /// A sender for updating our [`PublishStatus`].
168    ///
169    /// When our [`PublishStatus`] changes to [`UploadScheduled`](PublishStatus::UploadScheduled),
170    /// we can start publishing descriptors.
171    publish_status_tx: watch::Sender<PublishStatus>,
172    /// A channel for sending upload completion notifications.
173    ///
174    /// This channel is polled in the main loop of the reactor.
175    upload_task_complete_rx: mpsc::Receiver<TimePeriodUploadResult>,
176    /// A channel for receiving upload completion notifications.
177    ///
178    /// A copy of this sender is handed to each upload task.
179    upload_task_complete_tx: mpsc::Sender<TimePeriodUploadResult>,
180    /// A sender for notifying any pending upload tasks that the reactor is shutting down.
181    ///
182    /// Receivers can use this channel to find out when reactor is dropped.
183    ///
184    /// This is currently only used in [`upload_for_time_period`](Reactor::upload_for_time_period).
185    /// Any future background tasks can also use this channel to detect if the reactor is dropped.
186    ///
187    /// Closing this channel will cause any pending upload tasks to be dropped.
188    shutdown_tx: broadcast::Sender<Void>,
189    /// Path resolver for configuration files.
190    path_resolver: Arc<CfgPathResolver>,
191    /// Queue on which we receive messages from the [`PowManager`] telling us that a seed has
192    /// rotated and thus we need to republish the descriptor for a particular time period.
193    update_from_pow_manager_rx: mpsc::Receiver<TimePeriod>,
194}
195
196/// The immutable, shared state of the descriptor publisher reactor.
197#[derive(Clone)]
198struct Immutable<R: Runtime, M: Mockable> {
199    /// The runtime.
200    runtime: R,
201    /// Mockable state.
202    ///
203    /// This is used for launching circuits and for obtaining random number generators.
204    mockable: M,
205    /// The service for which we're publishing descriptors.
206    nickname: HsNickname,
207    /// The key manager,
208    keymgr: Arc<KeyMgr>,
209    /// A sender for updating the status of the onion service.
210    status_tx: PublisherStatusSender,
211    /// Proof-of-work state.
212    pow_manager: Arc<PowManager<R>>,
213    /// The directory to dump invalid HS descriptors to.
214    bad_hsdescs_dir: InstanceRawSubdir,
215}
216
217impl<R: Runtime, M: Mockable> Immutable<R, M> {
218    /// Create an [`AesOpeKey`] for generating revision counters for the descriptors associated
219    /// with the specified [`TimePeriod`].
220    ///
221    /// If the onion service is not running in offline mode, the key of the returned `AesOpeKey` is
222    /// the private part of the blinded identity key. Otherwise, the key is the private part of the
223    /// descriptor signing key.
224    ///
225    /// Returns an error if the service is running in offline mode and the descriptor signing
226    /// keypair of the specified `period` is not available.
227    //
228    // TODO (#1194): we don't support "offline" mode (yet), so this always returns an AesOpeKey
229    // built from the blinded id key
230    fn create_ope_key(&self, period: TimePeriod) -> Result<AesOpeKey, FatalError> {
231        let ope_key = match read_blind_id_keypair(&self.keymgr, &self.nickname, period)? {
232            Some(key) => {
233                let key: ed25519::ExpandedKeypair = key.into();
234                key.to_secret_key_bytes()[0..32]
235                    .try_into()
236                    .expect("Wrong length on slice")
237            }
238            None => {
239                // TODO (#1194): we don't support externally provisioned keys (yet), so this branch
240                // is unreachable (for now).
241                let desc_sign_key_spec =
242                    DescSigningKeypairSpecifier::new(self.nickname.clone(), period);
243                let key: ed25519::Keypair = self
244                    .keymgr
245                    .get::<HsDescSigningKeypair>(&desc_sign_key_spec)?
246                    // TODO (#1194): internal! is not the right type for this error (we need an
247                    // error type for the case where a hidden service running in offline mode has
248                    // run out of its pre-previsioned keys).
249                    //
250                    // This will be addressed when we add support for offline hs_id mode
251                    .ok_or_else(|| {
252                        internal!(
253                            "identity keys are offline, but descriptor signing key is unavailable?!"
254                        )
255                    })?
256                    .into();
257                key.to_bytes()
258            }
259        };
260
261        Ok(AesOpeKey::from_secret(&ope_key))
262    }
263
264    /// Generate a revision counter for a descriptor associated with the specified
265    /// [`TimePeriod`].
266    ///
267    /// Returns a revision counter generated according to the [encrypted time in period] scheme.
268    ///
269    /// [encrypted time in period]: https://spec.torproject.org/rend-spec/revision-counter-mgt.html#encrypted-time
270    fn generate_revision_counter(
271        &self,
272        params: &HsDirParams,
273        now: SystemTime,
274    ) -> Result<RevisionCounter, FatalError> {
275        // TODO: in the future, we might want to compute ope_key once per time period (as oppposed
276        // to each time we generate a new descriptor), for performance reasons.
277        let ope_key = self.create_ope_key(params.time_period())?;
278
279        // TODO: perhaps this should be moved to a new HsDirParams::offset_within_sr() function
280        let srv_start = params.start_of_shard_rand_period();
281        let offset = params.offset_within_srv_period(now).ok_or_else(|| {
282            internal!(
283                "current wallclock time not within SRV range?! (now={:?}, SRV_start={:?})",
284                now,
285                srv_start
286            )
287        })?;
288        let rev = ope_key.encrypt(offset);
289
290        Ok(RevisionCounter::from(rev))
291    }
292}
293
294/// Mockable state for the descriptor publisher reactor.
295///
296/// This enables us to mock parts of the [`Reactor`] for testing purposes.
297#[async_trait]
298pub(crate) trait Mockable: Clone + Send + Sync + Sized + 'static {
299    /// The type of random number generator.
300    type Rng: rand::Rng + rand::CryptoRng;
301
302    /// The type of client circuit.
303    type Tunnel: MockableDirTunnel;
304
305    /// Return a random number generator.
306    fn thread_rng(&self) -> Self::Rng;
307
308    /// Create a circuit of the specified `kind` to `target`.
309    async fn get_or_launch_hs_dir<T>(
310        &self,
311        netdir: &NetDir,
312        target: T,
313    ) -> Result<Self::Tunnel, tor_circmgr::Error>
314    where
315        T: CircTarget + Send + Sync;
316
317    /// Return an estimate-based value for how long we should allow a single
318    /// directory upload operation to complete.
319    ///
320    /// Includes circuit construction, stream opening, upload, and waiting for a
321    /// response.
322    fn estimate_upload_timeout(&self) -> Duration;
323}
324
325/// Mockable client circuit
326#[async_trait]
327pub(crate) trait MockableDirTunnel: Send + Sync {
328    /// The data stream type.
329    type DataStream: AsyncRead + AsyncWrite + Send + Unpin;
330
331    /// Start a new stream to the last relay in the circuit, using
332    /// a BEGIN_DIR cell.
333    async fn begin_dir_stream(&self) -> Result<Self::DataStream, tor_circmgr::Error>;
334
335    /// Try to get a SourceInfo for this circuit, for using it in a directory request.
336    fn source_info(&self) -> tor_proto::Result<Option<SourceInfo>>;
337}
338
339#[async_trait]
340impl MockableDirTunnel for ServiceOnionServiceDirTunnel {
341    type DataStream = tor_proto::client::stream::DataStream;
342
343    async fn begin_dir_stream(&self) -> Result<Self::DataStream, tor_circmgr::Error> {
344        Self::begin_dir_stream(self).await
345    }
346
347    fn source_info(&self) -> tor_proto::Result<Option<SourceInfo>> {
348        SourceInfo::from_tunnel(self)
349    }
350}
351
352/// The real version of the mockable state of the reactor.
353#[derive(Clone, From, Into)]
354pub(crate) struct Real<R: Runtime>(Arc<HsCircPool<R>>);
355
356#[async_trait]
357impl<R: Runtime> Mockable for Real<R> {
358    type Rng = rand::rngs::ThreadRng;
359    type Tunnel = ServiceOnionServiceDirTunnel;
360
361    fn thread_rng(&self) -> Self::Rng {
362        rand::rng()
363    }
364
365    #[instrument(level = "trace", skip_all)]
366    async fn get_or_launch_hs_dir<T>(
367        &self,
368        netdir: &NetDir,
369        target: T,
370    ) -> Result<Self::Tunnel, tor_circmgr::Error>
371    where
372        T: CircTarget + Send + Sync,
373    {
374        self.0.get_or_launch_svc_dir(netdir, target).await
375    }
376
377    fn estimate_upload_timeout(&self) -> Duration {
378        use tor_circmgr::timeouts::Action;
379        let est_build = self.0.estimate_timeout(&Action::BuildCircuit { length: 4 });
380        let est_roundtrip = self.0.estimate_timeout(&Action::RoundTrip { length: 4 });
381        // We assume that in the worst case we'll have to wait for an entire
382        // circuit construction and two round-trips to the hsdir.
383        let est_total = est_build + est_roundtrip * 2;
384        // We always allow _at least_ this much time, in case our estimate is
385        // ridiculously low.
386        let min_timeout = Duration::from_secs(30);
387        max(est_total, min_timeout)
388    }
389}
390
391/// The mutable state of a [`Reactor`].
392struct Inner {
393    /// The onion service config.
394    config: Arc<OnionServiceConfigPublisherView>,
395    /// Watcher for key_dirs.
396    ///
397    /// Set to `None` if the reactor is not running, or if `watch_configuration` is false.
398    ///
399    /// The watcher is recreated whenever the `restricted_discovery.key_dirs` change.
400    file_watcher: Option<FileWatcher>,
401    /// The relevant time periods.
402    ///
403    /// This includes the current time period, as well as any other time periods we need to be
404    /// publishing descriptors for.
405    ///
406    /// This is empty until we fetch our first netdir in [`Reactor::run`].
407    time_periods: Vec<TimePeriodContext>,
408    /// Our most up to date netdir.
409    ///
410    /// This is initialized in [`Reactor::run`].
411    netdir: Option<Arc<NetDir>>,
412    /// The timestamp of our last upload.
413    ///
414    /// This is the time when the last update was _initiated_ (rather than completed), to prevent
415    /// the publisher from spawning multiple upload tasks at once in response to multiple external
416    /// events happening in quick succession, such as the IPT manager sending multiple IPT change
417    /// notifications in a short time frame (#1142), or an IPT change notification that's
418    /// immediately followed by a consensus change. Starting two upload tasks at once is not only
419    /// inefficient, but it also causes the publisher to generate two different descriptors with
420    /// the same revision counter (the revision counter is derived from the current timestamp),
421    /// which ultimately causes the slower upload task to fail (see #1142).
422    ///
423    /// Note: This is only used for deciding when to reschedule a rate-limited upload. It is _not_
424    /// used for retrying failed uploads (these are handled internally by
425    /// [`Reactor::upload_descriptor_with_retries`]).
426    last_uploaded: Option<Instant>,
427    /// A max-heap containing the time periods for which we need to reupload the descriptor.
428    // TODO: we are currently reuploading more than nececessary.
429    // Ideally, this shouldn't contain contain duplicate TimePeriods,
430    // because we only need to retain the latest reupload time for each time period.
431    //
432    // Currently, if, for some reason, we upload the descriptor multiple times for the same TP,
433    // we will end up with multiple ReuploadTimer entries for that TP,
434    // each of which will (eventually) result in a reupload.
435    //
436    // TODO: maybe this should just be a HashMap<TimePeriod, Instant>
437    //
438    // See https://gitlab.torproject.org/tpo/core/arti/-/merge_requests/1971#note_2994950
439    reupload_timers: BinaryHeap<ReuploadTimer>,
440    /// The restricted discovery authorized clients.
441    ///
442    /// `None`, unless the service is running in restricted discovery mode.
443    authorized_clients: Option<Arc<RestrictedDiscoveryKeys>>,
444}
445
446/// The part of the reactor state that changes with every time period.
447struct TimePeriodContext {
448    /// The HsDir params.
449    params: HsDirParams,
450    /// The HsDirs to use in this time period.
451    ///
452    // We keep a list of `RelayIds` because we can't store a `Relay<'_>` inside the reactor
453    // (the lifetime of a relay is tied to the lifetime of its corresponding `NetDir`. To
454    // store `Relay<'_>`s in the reactor, we'd need a way of atomically swapping out both the
455    // `NetDir` and the cached relays, and to convince Rust what we're doing is sound)
456    hs_dirs: Vec<(RelayIds, DescriptorStatus)>,
457    /// The revision counter of the last successful upload, if any.
458    last_successful: Option<RevisionCounter>,
459    /// The outcome of the last upload, if any.
460    upload_results: Vec<HsDirUploadStatus>,
461}
462
463impl TimePeriodContext {
464    /// Create a new `TimePeriodContext`.
465    ///
466    /// Any of the specified `old_hsdirs` also present in the new list of HsDirs
467    /// (returned by `NetDir::hs_dirs_upload`) will have their `DescriptorStatus` preserved.
468    fn new<'r>(
469        params: HsDirParams,
470        blind_id: HsBlindId,
471        netdir: &Arc<NetDir>,
472        old_hsdirs: impl Iterator<Item = &'r (RelayIds, DescriptorStatus)>,
473        old_upload_results: Vec<HsDirUploadStatus>,
474    ) -> Result<Self, FatalError> {
475        let period = params.time_period();
476        let hs_dirs = Self::compute_hsdirs(period, blind_id, netdir, old_hsdirs)?;
477        let upload_results = old_upload_results
478            .into_iter()
479            .filter(|res|
480                // Check if the HsDir of this result still exists
481                hs_dirs
482                    .iter()
483                    .any(|(relay_ids, _status)| relay_ids == &res.relay_ids))
484            .collect();
485
486        Ok(Self {
487            params,
488            hs_dirs,
489            last_successful: None,
490            upload_results,
491        })
492    }
493
494    /// Recompute the HsDirs for this time period.
495    fn compute_hsdirs<'r>(
496        period: TimePeriod,
497        blind_id: HsBlindId,
498        netdir: &Arc<NetDir>,
499        mut old_hsdirs: impl Iterator<Item = &'r (RelayIds, DescriptorStatus)>,
500    ) -> Result<Vec<(RelayIds, DescriptorStatus)>, FatalError> {
501        let hs_dirs = netdir.hs_dirs_upload(blind_id, period)?;
502
503        Ok(hs_dirs
504            .map(|hs_dir| {
505                let mut builder = RelayIds::builder();
506                if let Some(ed_id) = hs_dir.ed_identity() {
507                    builder.ed_identity(*ed_id);
508                }
509
510                if let Some(rsa_id) = hs_dir.rsa_identity() {
511                    builder.rsa_identity(*rsa_id);
512                }
513
514                let relay_id = builder.build().unwrap_or_else(|_| RelayIds::empty());
515
516                // Have we uploaded the descriptor to thiw relay before? If so, we don't need to
517                // reupload it unless it was already dirty and due for a reupload.
518                let status = match old_hsdirs.find(|(id, _)| *id == relay_id) {
519                    Some((_, status)) => *status,
520                    None => DescriptorStatus::Dirty,
521                };
522
523                (relay_id, status)
524            })
525            .collect::<Vec<_>>())
526    }
527
528    /// Mark the descriptor dirty for all HSDirs of this time period.
529    fn mark_all_dirty(&mut self) {
530        self.hs_dirs
531            .iter_mut()
532            .for_each(|(_relay_id, status)| *status = DescriptorStatus::Dirty);
533    }
534
535    /// Update the upload result for this time period.
536    fn set_upload_results(&mut self, upload_results: Vec<HsDirUploadStatus>) {
537        self.upload_results = upload_results;
538    }
539}
540
541/// An error that occurs while trying to upload a descriptor.
542#[derive(Clone, Debug, thiserror::Error)]
543#[non_exhaustive]
544pub enum UploadError {
545    /// An error that has occurred after we have contacted a directory cache and made a circuit to it.
546    #[error("descriptor upload request failed: {}", _0.error)]
547    Request(#[from] RequestFailedError),
548
549    /// Failed to establish circuit to hidden service directory
550    #[error("could not build circuit to HsDir")]
551    Circuit(#[from] tor_circmgr::Error),
552
553    /// Failed to establish stream to hidden service directory
554    #[error("failed to establish directory stream to HsDir")]
555    Stream(#[source] tor_circmgr::Error),
556
557    /// An internal error.
558    #[error("Internal error")]
559    Bug(#[from] tor_error::Bug),
560}
561define_asref_dyn_std_error!(UploadError);
562
563impl UploadError {
564    /// Return true if this error is one that we should report as a suspicious event,
565    /// along with the dirserver, and description of the relevant document.
566    pub(crate) fn should_report_as_suspicious(&self) -> bool {
567        match self {
568            UploadError::Request(e) => e.error.should_report_as_suspicious_if_anon(),
569            UploadError::Circuit(_) => false, // TODO prop360
570            UploadError::Stream(_) => false,  // TODO prop360
571            UploadError::Bug(_) => false,
572        }
573    }
574}
575
576impl<R: Runtime, M: Mockable> Reactor<R, M> {
577    /// Create a new `Reactor`.
578    #[allow(clippy::too_many_arguments)]
579    pub(super) fn new(
580        runtime: R,
581        nickname: HsNickname,
582        dir_provider: Arc<dyn NetDirProvider>,
583        mockable: M,
584        config: &OnionServiceConfig,
585        ipt_watcher: IptsPublisherView,
586        config_rx: watch::Receiver<Arc<OnionServiceConfig>>,
587        status_tx: PublisherStatusSender,
588        keymgr: Arc<KeyMgr>,
589        path_resolver: Arc<CfgPathResolver>,
590        pow_manager: Arc<PowManager<R>>,
591        update_from_pow_manager_rx: mpsc::Receiver<TimePeriod>,
592        bad_hsdescs_dir: InstanceRawSubdir,
593    ) -> Self {
594        /// The maximum size of the upload completion notifier channel.
595        ///
596        /// The channel we use this for is a futures::mpsc channel, which has a capacity of
597        /// `UPLOAD_CHAN_BUF_SIZE + num-senders`. We don't need the buffer size to be non-zero, as
598        /// each sender will send exactly one message.
599        const UPLOAD_CHAN_BUF_SIZE: usize = 0;
600
601        // Internally-generated instructions, no need for mq.
602        let (upload_task_complete_tx, upload_task_complete_rx) =
603            mpsc_channel_no_memquota(UPLOAD_CHAN_BUF_SIZE);
604
605        let (publish_status_tx, publish_status_rx) = watch::channel();
606        // Setting the buffer size to zero here is OK,
607        // since we never actually send anything on this channel.
608        let (shutdown_tx, _shutdown_rx) = broadcast::channel(0);
609
610        let authorized_clients =
611            Self::read_authorized_clients(&config.restricted_discovery, &path_resolver);
612
613        // Create a channel for watching for changes in the configured
614        // restricted_discovery.key_dirs.
615        let (key_dirs_tx, key_dirs_rx) = file_watcher::channel();
616
617        let imm = Immutable {
618            runtime,
619            mockable,
620            nickname,
621            keymgr,
622            status_tx,
623            pow_manager,
624            bad_hsdescs_dir,
625        };
626
627        let inner = Inner {
628            time_periods: vec![],
629            config: Arc::new(config.into()),
630            file_watcher: None,
631            netdir: None,
632            last_uploaded: None,
633            reupload_timers: Default::default(),
634            authorized_clients,
635        };
636
637        Self {
638            imm: Arc::new(imm),
639            inner: Arc::new(Mutex::new(inner)),
640            dir_provider,
641            ipt_watcher,
642            config_rx,
643            key_dirs_rx,
644            key_dirs_tx,
645            publish_status_rx,
646            publish_status_tx,
647            upload_task_complete_rx,
648            upload_task_complete_tx,
649            shutdown_tx,
650            path_resolver,
651            update_from_pow_manager_rx,
652        }
653    }
654
655    /// Start the reactor.
656    ///
657    /// Under normal circumstances, this function runs indefinitely.
658    ///
659    /// Note: this also spawns the "reminder task" that we use to reschedule uploads whenever an
660    /// upload fails or is rate-limited.
661    pub(super) async fn run(mut self) -> Result<(), FatalError> {
662        debug!(nickname=%self.imm.nickname, "starting descriptor publisher reactor");
663
664        {
665            let netdir = self
666                .dir_provider
667                .wait_for_netdir(Timeliness::Timely)
668                .await?;
669            let time_periods = self.compute_time_periods(&netdir, &[])?;
670
671            let mut inner = self.inner.lock().expect("poisoned lock");
672
673            inner.netdir = Some(netdir);
674            inner.time_periods = time_periods;
675        }
676
677        // Create the initial key_dirs watcher.
678        self.update_file_watcher();
679
680        loop {
681            match self.run_once().await {
682                Ok(ShutdownStatus::Continue) => continue,
683                Ok(ShutdownStatus::Terminate) => {
684                    debug!(nickname=%self.imm.nickname, "descriptor publisher is shutting down!");
685
686                    self.imm.status_tx.send_shutdown();
687                    return Ok(());
688                }
689                Err(e) => {
690                    error_report!(
691                        e,
692                        "HS service {}: descriptor publisher crashed!",
693                        self.imm.nickname
694                    );
695
696                    self.imm.status_tx.send_broken(e.clone());
697
698                    return Err(e);
699                }
700            }
701        }
702    }
703
704    /// Run one iteration of the reactor loop.
705    async fn run_once(&mut self) -> Result<ShutdownStatus, FatalError> {
706        let mut netdir_events = self.dir_provider.events();
707
708        // Note: TrackingNow tracks the values it is compared with.
709        // This is equivalent to sleeping for (until - now) units of time,
710        let upload_rate_lim: TrackingNow = TrackingNow::now(&self.imm.runtime);
711        if let PublishStatus::RateLimited(until) = self.status() {
712            if upload_rate_lim > until {
713                // We are no longer rate-limited
714                self.expire_rate_limit().await?;
715            }
716        }
717
718        let reupload_tracking = TrackingNow::now(&self.imm.runtime);
719        let mut reupload_periods = vec![];
720        {
721            let mut inner = self.inner.lock().expect("poisoned lock");
722            let inner = &mut *inner;
723            while let Some(reupload) = inner.reupload_timers.peek().copied() {
724                // First, extract all the timeouts that already elapsed.
725                if reupload.when <= reupload_tracking {
726                    inner.reupload_timers.pop();
727                    reupload_periods.push(reupload.period);
728                } else {
729                    // We are not ready to schedule any more reuploads.
730                    //
731                    // How much we need to sleep is implicitly
732                    // tracked in reupload_tracking (through
733                    // the TrackingNow implementation)
734                    break;
735                }
736            }
737        }
738
739        // Check if it's time to schedule any reuploads.
740        for period in reupload_periods {
741            if self.mark_dirty(&period) {
742                debug!(
743                    time_period=?period,
744                    "descriptor reupload timer elapsed; scheduling reupload",
745                );
746                self.update_publish_status_unless_rate_lim(PublishStatus::UploadScheduled)
747                    .await?;
748            }
749        }
750
751        select_biased! {
752            res = self.upload_task_complete_rx.next().fuse() => {
753                let Some(upload_res) = res else {
754                    return Ok(ShutdownStatus::Terminate);
755                };
756
757                self.handle_upload_results(upload_res);
758                self.upload_result_to_svc_status()?;
759            },
760            () = upload_rate_lim.wait_for_earliest(&self.imm.runtime).fuse() => {
761                self.expire_rate_limit().await?;
762            },
763            () = reupload_tracking.wait_for_earliest(&self.imm.runtime).fuse() => {
764                // Run another iteration, executing run_once again. This time, we will remove the
765                // expired reupload from self.reupload_timers, mark the descriptor dirty for all
766                // relevant HsDirs, and schedule the upload by setting our status to
767                // UploadScheduled.
768                return Ok(ShutdownStatus::Continue);
769            },
770            netdir_event = netdir_events.next().fuse() => {
771                let Some(netdir_event) = netdir_event else {
772                    debug!("netdir event stream ended");
773                    return Ok(ShutdownStatus::Terminate);
774                };
775
776                if !matches!(netdir_event, DirEvent::NewConsensus) {
777                    return Ok(ShutdownStatus::Continue);
778                };
779
780                // The consensus changed. Grab a new NetDir.
781                let netdir = match self.dir_provider.netdir(Timeliness::Timely) {
782                    Ok(y) => y,
783                    Err(e) => {
784                        error_report!(e, "HS service {}: netdir unavailable. Retrying...", self.imm.nickname);
785                        // Hopefully a netdir will appear in the future.
786                        // in the meantime, suspend operations.
787                        //
788                        // TODO (#1218): there is a bug here: we stop reading on our inputs
789                        // including eg publish_status_rx, but it is our job to log some of
790                        // these things.  While we are waiting for a netdir, all those messages
791                        // are "stuck"; they'll appear later, with misleading timestamps.
792                        //
793                        // Probably this should be fixed by moving the logging
794                        // out of the reactor, where it won't be blocked.
795                        self.dir_provider.wait_for_netdir(Timeliness::Timely)
796                            .await?
797                    }
798                };
799                let relevant_periods = netdir.hs_all_time_periods();
800                self.handle_consensus_change(netdir).await?;
801                expire_publisher_keys(
802                    &self.imm.keymgr,
803                    &self.imm.nickname,
804                    &relevant_periods,
805                ).unwrap_or_else(|e| {
806                    error_report!(e, "failed to remove expired keys");
807                });
808            }
809            update = self.ipt_watcher.await_update().fuse() => {
810                if self.handle_ipt_change(update).await? == ShutdownStatus::Terminate {
811                    return Ok(ShutdownStatus::Terminate);
812                }
813            },
814            config = self.config_rx.next().fuse() => {
815                let Some(config) = config else {
816                    return Ok(ShutdownStatus::Terminate);
817                };
818
819                self.handle_svc_config_change(&config).await?;
820            },
821            res = self.key_dirs_rx.next().fuse() => {
822                let Some(event) = res else {
823                    return Ok(ShutdownStatus::Terminate);
824                };
825
826                while let Some(_ignore) = self.key_dirs_rx.try_recv() {
827                    // Discard other events, so that we only reload once.
828                }
829
830                self.handle_key_dirs_change(event).await?;
831            }
832            should_upload = self.publish_status_rx.next().fuse() => {
833                let Some(should_upload) = should_upload else {
834                    return Ok(ShutdownStatus::Terminate);
835                };
836
837                // Our PublishStatus changed -- are we ready to publish?
838                if should_upload == PublishStatus::UploadScheduled {
839                    self.update_publish_status_unless_waiting(PublishStatus::Idle).await?;
840                    self.upload_all().await?;
841                }
842            }
843            update_tp_pow_seed = self.update_from_pow_manager_rx.next().fuse() => {
844                debug!("Update PoW seed for TP!");
845                let Some(time_period) = update_tp_pow_seed else {
846                    return Ok(ShutdownStatus::Terminate);
847                };
848                self.mark_dirty(&time_period);
849                self.upload_all().await?;
850            }
851        }
852
853        Ok(ShutdownStatus::Continue)
854    }
855
856    /// Returns the current status of the publisher
857    fn status(&self) -> PublishStatus {
858        *self.publish_status_rx.borrow()
859    }
860
861    /// Handle a batch of upload outcomes,
862    /// possibly updating the status of the descriptor for the corresponding HSDirs.
863    fn handle_upload_results(&self, results: TimePeriodUploadResult) {
864        let mut inner = self.inner.lock().expect("poisoned lock");
865        let inner = &mut *inner;
866
867        // Check which time period these uploads pertain to.
868        let period = inner
869            .time_periods
870            .iter_mut()
871            .find(|ctx| ctx.params.time_period() == results.time_period);
872
873        let Some(period) = period else {
874            // The uploads were for a time period that is no longer relevant, so we
875            // can ignore the result.
876            return;
877        };
878
879        // We will need to reupload this descriptor at some point, so we pick
880        // a random time between 60 minutes and 120 minutes in the future.
881        //
882        // See https://spec.torproject.org/rend-spec/deriving-keys.html#WHEN-HSDESC
883        let mut rng = self.imm.mockable.thread_rng();
884        // TODO SPEC: Control republish period using a consensus parameter?
885        let minutes = rng.gen_range_checked(60..=120).expect("low > high?!");
886        let duration = Duration::from_secs(minutes * 60);
887        let reupload_when = self.imm.runtime.now() + duration;
888        let time_period = period.params.time_period();
889
890        info!(
891            time_period=?time_period,
892            "reuploading descriptor in {}",
893            humantime::format_duration(duration),
894        );
895
896        inner.reupload_timers.push(ReuploadTimer {
897            period: time_period,
898            when: reupload_when,
899        });
900
901        let mut upload_results = vec![];
902        for upload_res in results.hsdir_result {
903            let relay = period
904                .hs_dirs
905                .iter_mut()
906                .find(|(relay_ids, _status)| relay_ids == &upload_res.relay_ids);
907
908            let Some((_relay, status)): Option<&mut (RelayIds, _)> = relay else {
909                // This HSDir went away, so the result doesn't matter.
910                // Continue processing the rest of the results
911                continue;
912            };
913
914            if upload_res.upload_res.is_ok() {
915                let update_last_successful = match period.last_successful {
916                    None => true,
917                    Some(counter) => counter <= upload_res.revision_counter,
918                };
919
920                if update_last_successful {
921                    period.last_successful = Some(upload_res.revision_counter);
922                    // TODO (#1098): Is it possible that this won't update the statuses promptly
923                    // enough. For example, it's possible for the reactor to see a Dirty descriptor
924                    // and start an upload task for a descriptor has already been uploaded (or is
925                    // being uploaded) in another task, but whose upload results have not yet been
926                    // processed.
927                    //
928                    // This is probably made worse by the fact that the statuses are updated in
929                    // batches (grouped by time period), rather than one by one as the upload tasks
930                    // complete (updating the status involves locking the inner mutex, and I wanted
931                    // to minimize the locking/unlocking overheads). I'm not sure handling the
932                    // updates in batches was the correct decision here.
933                    *status = DescriptorStatus::Clean;
934                }
935            }
936
937            upload_results.push(upload_res);
938        }
939
940        period.set_upload_results(upload_results);
941    }
942
943    /// Maybe update our list of HsDirs.
944    async fn handle_consensus_change(&mut self, netdir: Arc<NetDir>) -> Result<(), FatalError> {
945        trace!("the consensus has changed; recomputing HSDirs");
946
947        let _old: Option<Arc<NetDir>> = self.replace_netdir(netdir);
948
949        self.recompute_hs_dirs()?;
950        self.update_publish_status_unless_waiting(PublishStatus::UploadScheduled)
951            .await?;
952
953        // If the time period has changed, some of our upload results may now be irrelevant,
954        // so we might need to update our status (for example, if our uploads are
955        // for a no-longer-relevant time period, it means we might be able to update
956        // out status from "degraded" to "running")
957        self.upload_result_to_svc_status()?;
958
959        Ok(())
960    }
961
962    /// Recompute the HsDirs for all relevant time periods.
963    fn recompute_hs_dirs(&self) -> Result<(), FatalError> {
964        let mut inner = self.inner.lock().expect("poisoned lock");
965        let inner = &mut *inner;
966
967        let netdir = Arc::clone(
968            inner
969                .netdir
970                .as_ref()
971                .ok_or_else(|| internal!("started upload task without a netdir"))?,
972        );
973
974        // Update our list of relevant time periods.
975        let new_time_periods = self.compute_time_periods(&netdir, &inner.time_periods)?;
976        inner.time_periods = new_time_periods;
977
978        Ok(())
979    }
980
981    /// Compute the [`TimePeriodContext`]s for the time periods from the specified [`NetDir`].
982    ///
983    /// The specified `time_periods` are used to preserve the `DescriptorStatus` of the
984    /// HsDirs where possible.
985    fn compute_time_periods(
986        &self,
987        netdir: &Arc<NetDir>,
988        time_periods: &[TimePeriodContext],
989    ) -> Result<Vec<TimePeriodContext>, FatalError> {
990        netdir
991            .hs_all_time_periods()
992            .iter()
993            .map(|params| {
994                let period = params.time_period();
995                let blind_id_kp =
996                    read_blind_id_keypair(&self.imm.keymgr, &self.imm.nickname, period)?
997                        // Note: for now, read_blind_id_keypair cannot return Ok(None).
998                        // It's supposed to return Ok(None) if we're in offline hsid mode,
999                        // but that might change when we do #1194
1000                        .ok_or_else(|| internal!("offline hsid mode not supported"))?;
1001
1002                let blind_id: HsBlindIdKey = (&blind_id_kp).into();
1003
1004                // If our previous `TimePeriodContext`s also had an entry for `period`, we need to
1005                // preserve the `DescriptorStatus` of its HsDirs. This helps prevent unnecessarily
1006                // publishing the descriptor to the HsDirs that already have it (the ones that are
1007                // marked with DescriptorStatus::Clean).
1008                //
1009                // In other words, we only want to publish to those HsDirs that
1010                //   * are part of a new time period (which we have never published the descriptor
1011                //   for), or
1012                //   * have just been added to the ring of a time period we already knew about
1013                if let Some(ctx) = time_periods
1014                    .iter()
1015                    .find(|ctx| ctx.params.time_period() == period)
1016                {
1017                    TimePeriodContext::new(
1018                        params.clone(),
1019                        blind_id.into(),
1020                        netdir,
1021                        ctx.hs_dirs.iter(),
1022                        ctx.upload_results.clone(),
1023                    )
1024                } else {
1025                    // Passing an empty iterator here means all HsDirs in this TimePeriodContext
1026                    // will be marked as dirty, meaning we will need to upload our descriptor to them.
1027                    TimePeriodContext::new(
1028                        params.clone(),
1029                        blind_id.into(),
1030                        netdir,
1031                        iter::empty(),
1032                        vec![],
1033                    )
1034                }
1035            })
1036            .collect::<Result<Vec<TimePeriodContext>, FatalError>>()
1037    }
1038
1039    /// Replace the old netdir with the new, returning the old.
1040    fn replace_netdir(&self, new_netdir: Arc<NetDir>) -> Option<Arc<NetDir>> {
1041        self.inner
1042            .lock()
1043            .expect("poisoned lock")
1044            .netdir
1045            .replace(new_netdir)
1046    }
1047
1048    /// Replace our view of the service config with `new_config` if `new_config` contains changes
1049    /// that would cause us to generate a new descriptor.
1050    fn replace_config_if_changed(&self, new_config: Arc<OnionServiceConfigPublisherView>) -> bool {
1051        let mut inner = self.inner.lock().expect("poisoned lock");
1052        let old_config = &mut inner.config;
1053
1054        // The fields we're interested in haven't changed, so there's no need to update
1055        // `inner.config`.
1056        if *old_config == new_config {
1057            return false;
1058        }
1059
1060        let log_change = match (
1061            old_config.restricted_discovery.enabled,
1062            new_config.restricted_discovery.enabled,
1063        ) {
1064            (true, false) => Some("Disabling restricted discovery mode"),
1065            (false, true) => Some("Enabling restricted discovery mode"),
1066            _ => None,
1067        };
1068
1069        if let Some(msg) = log_change {
1070            info!(nickname=%self.imm.nickname, "{}", msg);
1071        }
1072
1073        let _old: Arc<OnionServiceConfigPublisherView> = std::mem::replace(old_config, new_config);
1074
1075        true
1076    }
1077
1078    /// Recreate the FileWatcher for watching the restricted discovery key_dirs.
1079    fn update_file_watcher(&self) {
1080        let mut inner = self.inner.lock().expect("poisoned lock");
1081        if inner.config.restricted_discovery.watch_configuration() {
1082            debug!("The restricted_discovery.key_dirs have changed, updating file watcher");
1083            let mut watcher = FileWatcher::builder(self.imm.runtime.clone());
1084
1085            let dirs = inner.config.restricted_discovery.key_dirs().clone();
1086
1087            watch_dirs(&mut watcher, &dirs, &self.path_resolver);
1088
1089            let watcher = watcher
1090                .start_watching(self.key_dirs_tx.clone())
1091                .map_err(|e| {
1092                    // TODO: update the publish status (see also the module-level TODO about this).
1093                    error_report!(e, "Cannot set file watcher");
1094                })
1095                .ok();
1096            inner.file_watcher = watcher;
1097        } else {
1098            if inner.file_watcher.is_some() {
1099                debug!("removing key_dirs watcher");
1100            }
1101            inner.file_watcher = None;
1102        }
1103    }
1104
1105    /// Read the intro points from `ipt_watcher`, and decide whether we're ready to start
1106    /// uploading.
1107    fn note_ipt_change(&self) -> PublishStatus {
1108        let mut ipts = self.ipt_watcher.borrow_for_publish();
1109        match ipts.ipts.as_mut() {
1110            Some(_ipts) => PublishStatus::UploadScheduled,
1111            None => PublishStatus::AwaitingIpts,
1112        }
1113    }
1114
1115    /// Update our list of introduction points.
1116    async fn handle_ipt_change(
1117        &mut self,
1118        update: Option<Result<(), crate::FatalError>>,
1119    ) -> Result<ShutdownStatus, FatalError> {
1120        trace!(nickname=%self.imm.nickname, "received IPT change notification from IPT manager");
1121        match update {
1122            Some(Ok(())) => {
1123                let should_upload = self.note_ipt_change();
1124                debug!(nickname=%self.imm.nickname, "the introduction points have changed");
1125
1126                self.mark_all_dirty();
1127                self.update_publish_status_unless_rate_lim(should_upload)
1128                    .await?;
1129                Ok(ShutdownStatus::Continue)
1130            }
1131            Some(Err(e)) => Err(e),
1132            None => {
1133                debug!(nickname=%self.imm.nickname, "received shut down signal from IPT manager");
1134                Ok(ShutdownStatus::Terminate)
1135            }
1136        }
1137    }
1138
1139    /// Update the `PublishStatus` of the reactor with `new_state`,
1140    /// unless the current state is `AwaitingIpts`.
1141    async fn update_publish_status_unless_waiting(
1142        &mut self,
1143        new_state: PublishStatus,
1144    ) -> Result<(), FatalError> {
1145        // Only update the state if we're not waiting for intro points.
1146        if self.status() != PublishStatus::AwaitingIpts {
1147            self.update_publish_status(new_state).await?;
1148        }
1149
1150        Ok(())
1151    }
1152
1153    /// Update the `PublishStatus` of the reactor with `new_state`,
1154    /// unless the current state is `RateLimited`.
1155    async fn update_publish_status_unless_rate_lim(
1156        &mut self,
1157        new_state: PublishStatus,
1158    ) -> Result<(), FatalError> {
1159        // We can't exit this state until the rate-limit expires.
1160        if !matches!(self.status(), PublishStatus::RateLimited(_)) {
1161            self.update_publish_status(new_state).await?;
1162        }
1163
1164        Ok(())
1165    }
1166
1167    /// Unconditionally update the `PublishStatus` of the reactor with `new_state`.
1168    async fn update_publish_status(&mut self, new_state: PublishStatus) -> Result<(), Bug> {
1169        let onion_status = match new_state {
1170            PublishStatus::Idle => None,
1171            PublishStatus::UploadScheduled
1172            | PublishStatus::AwaitingIpts
1173            | PublishStatus::RateLimited(_) => Some(State::Bootstrapping),
1174        };
1175
1176        if let Some(onion_status) = onion_status {
1177            self.imm.status_tx.send(onion_status, None);
1178        }
1179
1180        trace!(
1181            "publisher reactor status change: {:?} -> {:?}",
1182            self.status(),
1183            new_state
1184        );
1185
1186        self.publish_status_tx.send(new_state).await.map_err(
1187            |_: postage::sink::SendError<_>| internal!("failed to send upload notification?!"),
1188        )?;
1189
1190        Ok(())
1191    }
1192
1193    /// Update the onion svc status based on the results of the last descriptor uploads.
1194    fn upload_result_to_svc_status(&self) -> Result<(), FatalError> {
1195        let inner = self.inner.lock().expect("poisoned lock");
1196        let netdir = inner
1197            .netdir
1198            .as_ref()
1199            .ok_or_else(|| internal!("handling upload results without netdir?!"))?;
1200
1201        let (state, err) = upload_result_state(netdir, &inner.time_periods);
1202        self.imm.status_tx.send(state, err);
1203
1204        Ok(())
1205    }
1206
1207    /// Update the descriptors based on the config change.
1208    async fn handle_svc_config_change(
1209        &mut self,
1210        config: &OnionServiceConfig,
1211    ) -> Result<(), FatalError> {
1212        let new_config = Arc::new(config.into());
1213        if self.replace_config_if_changed(Arc::clone(&new_config)) {
1214            self.update_file_watcher();
1215            self.update_authorized_clients_if_changed();
1216
1217            info!(nickname=%self.imm.nickname, "Config has changed, generating a new descriptor");
1218            self.mark_all_dirty();
1219
1220            // Schedule an upload, unless we're still waiting for IPTs.
1221            self.update_publish_status_unless_waiting(PublishStatus::UploadScheduled)
1222                .await?;
1223        }
1224
1225        Ok(())
1226    }
1227
1228    /// Update the descriptors based on a restricted discovery key_dirs change.
1229    ///
1230    /// If the authorized clients from the [`RestrictedDiscoveryConfig`] have changed,
1231    /// this marks the descriptor as dirty for all time periods,
1232    /// and schedules a reupload.
1233    async fn handle_key_dirs_change(&mut self, event: FileEvent) -> Result<(), FatalError> {
1234        debug!("The configured key_dirs have changed");
1235        match event {
1236            FileEvent::Rescan | FileEvent::FileChanged => {
1237                // These events are handled in the same way, by re-reading the keys from disk
1238                // and republishing the descriptor if necessary
1239            }
1240            _ => return Err(internal!("file watcher event {event:?}").into()),
1241        };
1242
1243        // Update the file watcher, in case the change was triggered by a key_dir move.
1244        self.update_file_watcher();
1245
1246        if self.update_authorized_clients_if_changed() {
1247            self.mark_all_dirty();
1248
1249            // Schedule an upload, unless we're still waiting for IPTs.
1250            self.update_publish_status_unless_waiting(PublishStatus::UploadScheduled)
1251                .await?;
1252        }
1253
1254        Ok(())
1255    }
1256
1257    /// Recreate the authorized_clients based on the current config.
1258    ///
1259    /// Returns `true` if the authorized clients have changed.
1260    fn update_authorized_clients_if_changed(&mut self) -> bool {
1261        let mut inner = self.inner.lock().expect("poisoned lock");
1262        let authorized_clients =
1263            Self::read_authorized_clients(&inner.config.restricted_discovery, &self.path_resolver);
1264
1265        let clients = &mut inner.authorized_clients;
1266        let changed = clients.as_ref() != authorized_clients.as_ref();
1267
1268        if changed {
1269            info!("The restricted discovery mode authorized clients have changed");
1270            *clients = authorized_clients;
1271        }
1272
1273        changed
1274    }
1275
1276    /// Read the authorized `RestrictedDiscoveryKeys` from `config`.
1277    fn read_authorized_clients(
1278        config: &RestrictedDiscoveryConfig,
1279        path_resolver: &CfgPathResolver,
1280    ) -> Option<Arc<RestrictedDiscoveryKeys>> {
1281        let authorized_clients = config.read_keys(path_resolver);
1282
1283        if matches!(authorized_clients.as_ref(), Some(c) if c.is_empty()) {
1284            warn!(
1285                "Running in restricted discovery mode, but we have no authorized clients. Service will be unreachable"
1286            );
1287        }
1288
1289        authorized_clients.map(Arc::new)
1290    }
1291
1292    /// Mark the descriptor dirty for all time periods.
1293    fn mark_all_dirty(&self) {
1294        trace!("marking the descriptor dirty for all time periods");
1295
1296        self.inner
1297            .lock()
1298            .expect("poisoned lock")
1299            .time_periods
1300            .iter_mut()
1301            .for_each(|tp| tp.mark_all_dirty());
1302    }
1303
1304    /// Mark the descriptor dirty for the specified time period.
1305    ///
1306    /// Returns `true` if the specified period is still relevant, and `false` otherwise.
1307    fn mark_dirty(&self, period: &TimePeriod) -> bool {
1308        let mut inner = self.inner.lock().expect("poisoned lock");
1309        let period_ctx = inner
1310            .time_periods
1311            .iter_mut()
1312            .find(|tp| tp.params.time_period() == *period);
1313
1314        match period_ctx {
1315            Some(ctx) => {
1316                trace!(time_period=?period, "marking the descriptor dirty");
1317                ctx.mark_all_dirty();
1318                true
1319            }
1320            None => false,
1321        }
1322    }
1323
1324    /// Try to upload our descriptor to the HsDirs that need it.
1325    ///
1326    /// If we've recently uploaded some descriptors, we return immediately and schedule the upload
1327    /// to happen after [`UPLOAD_RATE_LIM_THRESHOLD`].
1328    ///
1329    /// Failed uploads are retried
1330    /// (see [`upload_descriptor_with_retries`](Reactor::upload_descriptor_with_retries)).
1331    ///
1332    /// If restricted discovery mode is enabled and there are no authorized clients,
1333    /// we abort the upload and set our status to [`State::Broken`].
1334    //
1335    // Note: a broken restricted discovery config won't prevent future uploads from being scheduled
1336    // (for example if the IPTs change),
1337    // which can can cause the publisher's status to oscillate between `Bootstrapping` and `Broken`.
1338    // TODO: we might wish to refactor the publisher to be more sophisticated about this.
1339    //
1340    /// For each current time period, we spawn a task that uploads the descriptor to
1341    /// all the HsDirs on the HsDir ring of that time period.
1342    /// Each task shuts down on completion, or when the reactor is dropped.
1343    ///
1344    /// Each task reports its upload results (`TimePeriodUploadResult`)
1345    /// via the `upload_task_complete_tx` channel.
1346    /// The results are received and processed in the main loop of the reactor.
1347    ///
1348    /// Returns an error if it fails to spawn a task, or if an internal error occurs.
1349    async fn upload_all(&mut self) -> Result<(), FatalError> {
1350        trace!("starting descriptor upload task...");
1351
1352        // Abort the upload entirely if we have an empty list of authorized clients
1353        let authorized_clients = match self.authorized_clients() {
1354            Ok(authorized_clients) => authorized_clients,
1355            Err(e) => {
1356                error_report!(e, "aborting upload");
1357                self.imm.status_tx.send_broken(e.clone());
1358
1359                // Returning an error would shut down the reactor, so we have to return Ok here.
1360                return Ok(());
1361            }
1362        };
1363
1364        let last_uploaded = self.inner.lock().expect("poisoned lock").last_uploaded;
1365        let now = self.imm.runtime.now();
1366        // Check if we should rate-limit this upload.
1367        if let Some(ts) = last_uploaded {
1368            let duration_since_upload = now.duration_since(ts);
1369
1370            if duration_since_upload < UPLOAD_RATE_LIM_THRESHOLD {
1371                return Ok(self.start_rate_limit(UPLOAD_RATE_LIM_THRESHOLD).await?);
1372            }
1373        }
1374
1375        let mut inner = self.inner.lock().expect("poisoned lock");
1376        let inner = &mut *inner;
1377
1378        let _ = inner.last_uploaded.insert(now);
1379
1380        for period_ctx in inner.time_periods.iter_mut() {
1381            let upload_task_complete_tx = self.upload_task_complete_tx.clone();
1382
1383            // Figure out which HsDirs we need to upload the descriptor to (some of them might already
1384            // have our latest descriptor, so we filter them out).
1385            let hs_dirs = period_ctx
1386                .hs_dirs
1387                .iter()
1388                .filter_map(|(relay_id, status)| {
1389                    if *status == DescriptorStatus::Dirty {
1390                        Some(relay_id.clone())
1391                    } else {
1392                        None
1393                    }
1394                })
1395                .collect::<Vec<_>>();
1396
1397            if hs_dirs.is_empty() {
1398                trace!("the descriptor is clean for all HSDirs. Nothing to do");
1399                return Ok(());
1400            }
1401
1402            let time_period = period_ctx.params.time_period();
1403            // This scope exists because rng is not Send, so it needs to fall out of scope before we
1404            // await anything.
1405            let netdir = Arc::clone(
1406                inner
1407                    .netdir
1408                    .as_ref()
1409                    .ok_or_else(|| internal!("started upload task without a netdir"))?,
1410            );
1411
1412            let imm = Arc::clone(&self.imm);
1413            let ipt_upload_view = self.ipt_watcher.upload_view();
1414            let config = Arc::clone(&inner.config);
1415            let authorized_clients = authorized_clients.clone();
1416
1417            trace!(nickname=%self.imm.nickname, time_period=?time_period,
1418                "spawning upload task"
1419            );
1420
1421            let params = period_ctx.params.clone();
1422            let shutdown_rx = self.shutdown_tx.subscribe();
1423
1424            // Spawn a task to upload the descriptor to all HsDirs of this time period.
1425            //
1426            // This task will shut down when the reactor is dropped (i.e. when shutdown_rx is
1427            // dropped).
1428            let _handle: () = self
1429                .imm
1430                .runtime
1431                .spawn(async move {
1432                    if let Err(e) = Self::upload_for_time_period(
1433                        hs_dirs,
1434                        &netdir,
1435                        config,
1436                        params,
1437                        Arc::clone(&imm),
1438                        ipt_upload_view.clone(),
1439                        authorized_clients.clone(),
1440                        upload_task_complete_tx,
1441                        shutdown_rx,
1442                    )
1443                    .await
1444                    {
1445                        error_report!(
1446                            e,
1447                            "descriptor upload failed for HS service {} and time period {:?}",
1448                            imm.nickname,
1449                            time_period
1450                        );
1451                    }
1452                })
1453                .map_err(|e| FatalError::from_spawn("upload_for_time_period task", e))?;
1454        }
1455
1456        Ok(())
1457    }
1458
1459    /// Upload the descriptor for the time period specified in `params`.
1460    ///
1461    /// Failed uploads are retried
1462    /// (see [`upload_descriptor_with_retries`](Reactor::upload_descriptor_with_retries)).
1463    #[allow(clippy::too_many_arguments)] // TODO: refactor
1464    async fn upload_for_time_period(
1465        hs_dirs: Vec<RelayIds>,
1466        netdir: &Arc<NetDir>,
1467        config: Arc<OnionServiceConfigPublisherView>,
1468        params: HsDirParams,
1469        imm: Arc<Immutable<R, M>>,
1470        ipt_upload_view: IptsPublisherUploadView,
1471        authorized_clients: Option<Arc<RestrictedDiscoveryKeys>>,
1472        mut upload_task_complete_tx: mpsc::Sender<TimePeriodUploadResult>,
1473        shutdown_rx: broadcast::Receiver<Void>,
1474    ) -> Result<(), FatalError> {
1475        let time_period = params.time_period();
1476        trace!(time_period=?time_period, "uploading descriptor to all HSDirs for this time period");
1477
1478        let hsdir_count = hs_dirs.len();
1479
1480        /// An error returned from an upload future.
1481        //
1482        // Exhaustive, because this is a private type.
1483        #[derive(Clone, Debug, thiserror::Error)]
1484        enum PublishError {
1485            /// The upload was aborted because there are no IPTs.
1486            ///
1487            /// This happens because of an inevitable TOCTOU race, where after being notified by
1488            /// the IPT manager that the IPTs have changed (via `self.ipt_watcher.await_update`),
1489            /// we find out there actually are no IPTs, so we can't build the descriptor.
1490            ///
1491            /// This is a special kind of error that interrupts the current upload task, and is
1492            /// logged at `debug!` level rather than `warn!` or `error!`.
1493            ///
1494            /// Ideally, this shouldn't happen very often (if at all).
1495            #[error("No IPTs")]
1496            NoIpts,
1497
1498            /// The reactor has shut down
1499            #[error("The reactor has shut down")]
1500            Shutdown,
1501
1502            /// An fatal error.
1503            #[error("{0}")]
1504            Fatal(#[from] FatalError),
1505        }
1506
1507        let max_hsdesc_len: usize = netdir
1508            .params()
1509            .hsdir_max_desc_size
1510            .try_into()
1511            .expect("Unable to convert positive int32 to usize!?");
1512
1513        let upload_results = futures::stream::iter(hs_dirs)
1514            .map(|relay_ids| {
1515                let netdir = netdir.clone();
1516                let config = Arc::clone(&config);
1517                let imm = Arc::clone(&imm);
1518                let ipt_upload_view = ipt_upload_view.clone();
1519                let authorized_clients = authorized_clients.clone();
1520                let params = params.clone();
1521                let mut shutdown_rx = shutdown_rx.clone();
1522
1523                let ed_id = relay_ids
1524                    .rsa_identity()
1525                    .map(|id| id.to_string())
1526                    .unwrap_or_else(|| "unknown".into());
1527                let rsa_id = relay_ids
1528                    .rsa_identity()
1529                    .map(|id| id.to_string())
1530                    .unwrap_or_else(|| "unknown".into());
1531
1532                async move {
1533                    let run_upload = |desc| async {
1534                        let Some(hsdir) = netdir.by_ids(&relay_ids) else {
1535                            // This should never happen (all of our relay_ids are from the stored
1536                            // netdir).
1537                            let err =
1538                                "tried to upload descriptor to relay not found in consensus?!";
1539                            warn!(
1540                                nickname=%imm.nickname, hsdir_id=%ed_id, hsdir_rsa_id=%rsa_id,
1541                                "{err}"
1542                            );
1543                            return Err(internal!("{err}").into());
1544                        };
1545
1546                        Self::upload_descriptor_with_retries(
1547                            desc,
1548                            &netdir,
1549                            &hsdir,
1550                            &ed_id,
1551                            &rsa_id,
1552                            Arc::clone(&imm),
1553                        )
1554                        .await
1555                    };
1556
1557                    // How long until we're supposed to time out?
1558                    let worst_case_end = imm.runtime.now() + OVERALL_UPLOAD_TIMEOUT;
1559                    // We generate a new descriptor before _each_ HsDir upload. This means each
1560                    // HsDir could, in theory, receive a different descriptor (not just in terms of
1561                    // revision-counters, but also with a different set of IPTs). It may seem like
1562                    // this could lead to some HsDirs being left with an outdated descriptor, but
1563                    // that's not the case: after the upload completes, the publisher will be
1564                    // notified by the ipt_watcher of the IPT change event (if there was one to
1565                    // begin with), which will trigger another upload job.
1566                    let hsdesc = {
1567                        // This scope is needed because the ipt_set MutexGuard is not Send, so it
1568                        // needs to fall out of scope before the await point below
1569                        let mut ipt_set = ipt_upload_view.borrow_for_publish();
1570
1571                        // If there are no IPTs, we abort the upload. At this point, we might have
1572                        // uploaded the descriptor to some, but not all, HSDirs from the specified
1573                        // time period.
1574                        //
1575                        // Returning an error here means the upload completion task is never
1576                        // notified of the outcome of any of these uploads (which means the
1577                        // descriptor is not marked clean). This is OK, because if we suddenly find
1578                        // out we have no IPTs, it means our built `hsdesc` has an outdated set of
1579                        // IPTs, so we need to go back to the main loop to wait for IPT changes,
1580                        // and generate a fresh descriptor anyway.
1581                        //
1582                        // Ideally, this shouldn't happen very often (if at all).
1583                        let Some(ipts) = ipt_set.ipts.as_mut() else {
1584                            return Err(PublishError::NoIpts);
1585                        };
1586
1587                        let hsdesc = {
1588                            trace!(
1589                                nickname=%imm.nickname, time_period=?time_period,
1590                                "building descriptor"
1591                            );
1592                            let mut rng = imm.mockable.thread_rng();
1593                            let mut key_rng = tor_llcrypto::rng::CautiousRng;
1594
1595                            // We're about to generate a new version of the descriptor,
1596                            // so let's generate a new revision counter.
1597                            let now = imm.runtime.wallclock();
1598                            let revision_counter = imm.generate_revision_counter(&params, now)?;
1599
1600                            build_sign(
1601                                &imm.keymgr,
1602                                &imm.pow_manager,
1603                                &config,
1604                                netdir.params(),
1605                                authorized_clients.as_deref(),
1606                                ipts,
1607                                time_period,
1608                                revision_counter,
1609                                &mut rng,
1610                                &mut key_rng,
1611                                imm.runtime.wallclock(),
1612                                max_hsdesc_len,
1613                            )?
1614                        };
1615
1616                        if let Err(e) =
1617                            ipt_set.note_publication_attempt(&imm.runtime, worst_case_end)
1618                        {
1619                            let wait = e.log_retry_max(&imm.nickname)?;
1620                            // TODO (#1226): retry instead of this
1621                            return Err(FatalError::Bug(internal!(
1622                                "ought to retry after {wait:?}, crashing instead"
1623                            ))
1624                            .into());
1625                        }
1626
1627                        hsdesc
1628                    };
1629
1630                    let VersionedDescriptor {
1631                        desc,
1632                        revision_counter,
1633                    } = hsdesc;
1634                    let desc: Arc<str> = desc.into();
1635
1636                    trace!(
1637                        nickname=%imm.nickname, time_period=?time_period,
1638                        revision_counter=?revision_counter,
1639                        "generated new descriptor for time period",
1640                    );
1641
1642                    // (Actually launch the upload attempt. No timeout is needed
1643                    // here, since the backoff::Runner code will handle that for us.)
1644                    let upload_res: UploadResult = select_biased! {
1645                        shutdown = shutdown_rx.next().fuse() => {
1646                            // This will always be None, since Void is uninhabited.
1647                            let _: Option<Void> = shutdown;
1648
1649                            // It looks like the reactor has shut down,
1650                            // so there is no point in uploading the descriptor anymore.
1651                            //
1652                            // Let's shut down the upload task too.
1653                            trace!(
1654                                nickname=%imm.nickname, time_period=?time_period,
1655                                "upload task received shutdown signal"
1656                            );
1657
1658                            return Err(PublishError::Shutdown);
1659                        },
1660                        res = run_upload(desc.clone()).fuse() => res,
1661                    };
1662
1663                    // Note: UploadResult::Failure is only returned when
1664                    // upload_descriptor_with_retries fails, i.e. if all our retry
1665                    // attempts have failed
1666                    Ok(HsDirUploadStatus {
1667                        relay_ids,
1668                        upload_res,
1669                        revision_counter,
1670                    })
1671                }
1672            })
1673            // This fails to compile unless the stream is boxed. See https://github.com/rust-lang/rust/issues/104382
1674            .boxed()
1675            .buffer_unordered(MAX_CONCURRENT_UPLOADS)
1676            .try_collect::<Vec<_>>()
1677            .await;
1678
1679        let upload_results = match upload_results {
1680            Ok(v) => v,
1681            Err(PublishError::Fatal(e)) => return Err(e),
1682            Err(PublishError::NoIpts) => {
1683                debug!(
1684                    nickname=%imm.nickname, time_period=?time_period,
1685                     "no introduction points; skipping upload"
1686                );
1687
1688                return Ok(());
1689            }
1690            Err(PublishError::Shutdown) => {
1691                debug!(
1692                    nickname=%imm.nickname, time_period=?time_period,
1693                     "the reactor has shut down; aborting upload"
1694                );
1695
1696                return Ok(());
1697            }
1698        };
1699
1700        let (succeeded, _failed): (Vec<_>, Vec<_>) = upload_results
1701            .iter()
1702            .partition(|res| res.upload_res.is_ok());
1703
1704        debug!(
1705            nickname=%imm.nickname, time_period=?time_period,
1706            "descriptor uploaded successfully to {}/{} HSDirs",
1707            succeeded.len(), hsdir_count
1708        );
1709
1710        if upload_task_complete_tx
1711            .send(TimePeriodUploadResult {
1712                time_period,
1713                hsdir_result: upload_results,
1714            })
1715            .await
1716            .is_err()
1717        {
1718            return Err(internal!(
1719                "failed to notify reactor of upload completion (reactor shut down)"
1720            )
1721            .into());
1722        }
1723
1724        Ok(())
1725    }
1726
1727    /// Upload a descriptor to the specified HSDir.
1728    ///
1729    /// If an upload fails, this returns an `Err`. This function does not handle retries. It is up
1730    /// to the caller to retry on failure.
1731    ///
1732    /// This function does not handle timeouts.
1733    async fn upload_descriptor(
1734        hsdesc: Arc<str>,
1735        netdir: &Arc<NetDir>,
1736        hsdir: &Relay<'_>,
1737        imm: Arc<Immutable<R, M>>,
1738    ) -> Result<(), UploadError> {
1739        let request = HsDescUploadRequest::new(hsdesc);
1740
1741        trace!(nickname=%imm.nickname, hsdir_id=%hsdir.id(), hsdir_rsa_id=%hsdir.rsa_id(),
1742            "starting descriptor upload",
1743        );
1744
1745        let tunnel = imm
1746            .mockable
1747            .get_or_launch_hs_dir(netdir, OwnedCircTarget::from_circ_target(hsdir))
1748            .await?;
1749        let source: Option<SourceInfo> = tunnel
1750            .source_info()
1751            .map_err(into_internal!("Couldn't get SourceInfo for circuit"))?;
1752
1753        let mut stream = tunnel
1754            .begin_dir_stream()
1755            .await
1756            .map_err(UploadError::Stream)?;
1757
1758        let _response: String = send_request(&imm.runtime, &request, &mut stream, source)
1759            .await
1760            .map_err(|dir_error| -> UploadError {
1761                match dir_error {
1762                    DirClientError::RequestFailed(e) => e.into(),
1763                    DirClientError::CircMgr(e) => into_internal!(
1764                        "tor-dirclient complains about circmgr going wrong but we gave it a stream"
1765                    )(e)
1766                    .into(),
1767                    e => into_internal!("unexpected error")(e).into(),
1768                }
1769            })?
1770            .into_output_string()?; // This returns an error if we received an error response
1771
1772        Ok(())
1773    }
1774
1775    /// Upload a descriptor to the specified HSDir, retrying if appropriate.
1776    ///
1777    /// Any failed uploads are retried according to a [`PublisherBackoffSchedule`].
1778    /// Each failed upload is retried until it succeeds, or until the overall timeout specified
1779    /// by [`BackoffSchedule::overall_timeout`] elapses. Individual attempts are timed out
1780    /// according to the [`BackoffSchedule::single_attempt_timeout`].
1781    /// This function gives up after the overall timeout elapses,
1782    /// declaring the upload a failure, and never retrying it again.
1783    ///
1784    /// See also [`BackoffSchedule`].
1785    async fn upload_descriptor_with_retries(
1786        hsdesc: Arc<str>,
1787        netdir: &Arc<NetDir>,
1788        hsdir: &Relay<'_>,
1789        ed_id: &str,
1790        rsa_id: &str,
1791        imm: Arc<Immutable<R, M>>,
1792    ) -> UploadResult {
1793        /// The base delay to use for the backoff schedule.
1794        const BASE_DELAY_MSEC: u32 = 1000;
1795        let schedule = PublisherBackoffSchedule {
1796            retry_delay: RetryDelay::from_msec(BASE_DELAY_MSEC),
1797            mockable: imm.mockable.clone(),
1798        };
1799
1800        let runner = Runner::new(
1801            "upload a hidden service descriptor".into(),
1802            schedule.clone(),
1803            imm.runtime.clone(),
1804        );
1805
1806        let fallible_op = || async {
1807            let r = Self::upload_descriptor(hsdesc.clone(), netdir, hsdir, Arc::clone(&imm)).await;
1808
1809            if let Err(e) = &r {
1810                if e.should_report_as_suspicious() {
1811                    // Note that not every protocol violation is suspicious:
1812                    // we only warn on the protocol violations that look like attempts
1813                    // to do a traffic tagging attack via hsdir inflation.
1814                    // (See proposal 360.)
1815                    warn_report!(
1816                        e,
1817                        "Suspicious error while uploading descriptor to {}/{}",
1818                        ed_id,
1819                        rsa_id
1820                    );
1821                }
1822
1823                if let UploadError::Request(req_err) = e {
1824                    if let RequestError::HttpStatus(status, _msg) = &req_err.error {
1825                        if (400..500).contains(status) {
1826                            // This should never happen (it's a bug if it does!)
1827                            log_ratelim!(
1828                                "The peer says our HS descriptor upload request was invalid. This is a bug (maybe theirs; maybe ours)";
1829                                Err::<(), _>(e.clone());
1830                            );
1831                        }
1832
1833                        // C Tor HsDirs return a 400 status code if they deem
1834                        // the descriptor to be invalid.
1835                        if *status == 400 {
1836                            let res = dump_invalid_hsdesc(&hsdesc, &imm);
1837
1838                            log_ratelim!(
1839                                "Dumping rejected HS descriptor to {}",
1840                                imm.bad_hsdescs_dir.as_path().display_lossy();
1841                                res;
1842                            );
1843
1844                            // TODO: we might want to garbage-collect this at some point
1845                            // (e.g. if the mtime of the last dumped
1846                            // bad descriptor is far enough into the past?)
1847                        }
1848                    }
1849                }
1850            }
1851            r
1852        };
1853
1854        let outcome: Result<(), BackoffError<UploadError>> = runner.run(fallible_op).await;
1855        match outcome {
1856            Ok(()) => {
1857                debug!(
1858                    nickname=%imm.nickname, hsdir_id=%ed_id, hsdir_rsa_id=%rsa_id,
1859                    "successfully uploaded descriptor to HSDir",
1860                );
1861
1862                Ok(())
1863            }
1864            Err(e) => {
1865                warn_report!(
1866                    e,
1867                    "failed to upload descriptor for service {} (hsdir_id={}, hsdir_rsa_id={})",
1868                    imm.nickname,
1869                    ed_id,
1870                    rsa_id
1871                );
1872
1873                Err(e.into())
1874            }
1875        }
1876    }
1877
1878    /// Stop publishing descriptors until the specified delay elapses.
1879    async fn start_rate_limit(&mut self, delay: Duration) -> Result<(), Bug> {
1880        if !matches!(self.status(), PublishStatus::RateLimited(_)) {
1881            debug!(
1882                "We are rate-limited for {}; pausing descriptor publication",
1883                humantime::format_duration(delay)
1884            );
1885            let until = self.imm.runtime.now() + delay;
1886            self.update_publish_status(PublishStatus::RateLimited(until))
1887                .await?;
1888        }
1889
1890        Ok(())
1891    }
1892
1893    /// Handle the upload rate-limit being lifted.
1894    async fn expire_rate_limit(&mut self) -> Result<(), Bug> {
1895        debug!("We are no longer rate-limited; resuming descriptor publication");
1896        self.update_publish_status(PublishStatus::UploadScheduled)
1897            .await?;
1898        Ok(())
1899    }
1900
1901    /// Return the authorized clients, if restricted mode is enabled.
1902    ///
1903    /// Returns `Ok(None)` if restricted discovery mode is disabled.
1904    ///
1905    /// Returns an error if restricted discovery mode is enabled, but the client list is empty.
1906    #[cfg_attr(
1907        not(feature = "restricted-discovery"),
1908        allow(clippy::unnecessary_wraps)
1909    )]
1910    fn authorized_clients(&self) -> Result<Option<Arc<RestrictedDiscoveryKeys>>, FatalError> {
1911        cfg_if::cfg_if! {
1912            if #[cfg(feature = "restricted-discovery")] {
1913                let authorized_clients = self
1914                    .inner
1915                    .lock()
1916                    .expect("poisoned lock")
1917                    .authorized_clients
1918                    .clone();
1919
1920                if authorized_clients.as_ref().as_ref().map(|v| v.is_empty()).unwrap_or_default() {
1921                    return Err(FatalError::RestrictedDiscoveryNoClients);
1922                }
1923
1924                Ok(authorized_clients)
1925            } else {
1926                Ok(None)
1927            }
1928        }
1929    }
1930}
1931
1932/// Dump an invalid HS descriptor to disk, for debugging purposes.
1933///
1934/// The descriptor will get written to the state directory,
1935/// in `hss/<HS_NICKNAME>/bad_hsdescs/current`.
1936///
1937// TODO(#2760): this is a quick and dirty hack for debugging #2760.
1938// We may want to eventually replace this with a more principled
1939// "invalid descriptor" dumping utility, like the one C Tor has
1940// in dirparse/unparseable.c.
1941fn dump_invalid_hsdesc<R, M>(
1942    hsdesc: &Arc<str>,
1943    imm: &Arc<Immutable<R, M>>,
1944) -> Result<(), HsDescDumpError>
1945where
1946    R: Runtime,
1947    M: Mockable,
1948{
1949    use rand::distr::{Alphanumeric, SampleString};
1950    // If multiple upload tasks have an invalid
1951    // descriptor, they will all race to write to "filename.tmp",
1952    // so we need a unique id for the temporary file
1953    // (TODO: maybe it would be useful to have a FileAccess API for this?):
1954    //
1955    // Case-insensitive file systems are going to have a higher risk
1956    // of tmpname collisions for the tasks trying to write an invalid descriptor,
1957    // but we can try mitigate that by generating a relatively long string
1958    // (in practice, the chances of this happening are going to be very slim).
1959    let mut rng = imm.mockable.thread_rng();
1960    let tmpname = Alphanumeric.sample_string(&mut rng, 16);
1961    imm.bad_hsdescs_dir
1962        .write_and_replace(&tmpname, hsdesc.as_ref())?;
1963
1964    let tmp_path = imm.bad_hsdescs_dir.as_path().join(tmpname);
1965    let target = imm.bad_hsdescs_dir.as_path().join("current");
1966
1967    // This rename() bypasses the fs-mistrust checks, but I believe that's okay,
1968    // because the write_and_replace() above already checked the permissions
1969    // of the parent dir.
1970    let res = std::fs::rename(&tmp_path, &target).map_err(|err| HsDescDumpError::Rename {
1971        filename: target,
1972        err: Arc::new(err),
1973    });
1974
1975    if res.is_err() {
1976        // Try to at least clean up the temp file...
1977        let _ = std::fs::remove_file(tmp_path);
1978    }
1979
1980    Ok(())
1981}
1982
1983/// An error returned when trying to dump an invalid HS desc to disk.
1984#[derive(Clone, Debug, thiserror::Error)]
1985enum HsDescDumpError {
1986    /// We encountered a problem while inspecting or creating a directory.
1987    #[error("Problem accessing filesystem")]
1988    FsMistrust(#[from] fs_mistrust::Error),
1989
1990    /// We encountered an error while attempting an IO operation on a file.
1991    #[error(r#"IO error while attempting to rename file to {}"#, filename.anonymize_home())]
1992    Rename {
1993        /// The file that we were trying to modify or inspect
1994        filename: PathBuf,
1995        /// The error that we got when trying to perform the operation.
1996        #[source]
1997        err: Arc<std::io::Error>,
1998    },
1999}
2000
2001/// Try to expand a path, logging a warning on failure.
2002fn maybe_expand_path(p: &CfgPath, r: &CfgPathResolver) -> Option<PathBuf> {
2003    // map_err returns unit for clarity
2004    #[allow(clippy::unused_unit, clippy::semicolon_if_nothing_returned)]
2005    p.path(r)
2006        .map_err(|e| {
2007            tor_error::warn_report!(e, "invalid path");
2008            ()
2009        })
2010        .ok()
2011}
2012
2013/// Add `path` to the specified `watcher`.
2014macro_rules! watch_path {
2015    ($watcher:expr, $path:expr, $watch_fn:ident, $($watch_fn_args:expr,)*) => {{
2016        if let Err(e) = $watcher.$watch_fn(&$path, $($watch_fn_args)*) {
2017            warn_report!(e, "failed to watch path {:?}", $path);
2018        } else {
2019            debug!("watching path {:?}", $path);
2020        }
2021    }}
2022}
2023
2024/// Add the specified directories to the watcher.
2025fn watch_dirs<R: Runtime>(
2026    watcher: &mut FileWatcherBuilder<R>,
2027    dirs: &DirectoryKeyProviderList,
2028    path_resolver: &CfgPathResolver,
2029) {
2030    for path in dirs {
2031        let path = path.path();
2032        let Some(path) = maybe_expand_path(path, path_resolver) else {
2033            warn!("failed to expand key_dir path {:?}", path);
2034            continue;
2035        };
2036
2037        // If the path doesn't exist, the notify watcher will return an error if we attempt to watch it,
2038        // so we skip over paths that don't exist at this time
2039        // (this obviously suffers from a TOCTOU race, but most of the time,
2040        // it is good enough at preventing the watcher from failing to watch.
2041        // If the race *does* happen it is not disastrous, i.e. the reactor won't crash,
2042        // but it will fail to set the watcher).
2043        if matches!(path.try_exists(), Ok(true)) {
2044            watch_path!(watcher, &path, watch_dir, "auth",);
2045        }
2046        // FileWatcher::watch_path causes the parent dir of the path to be watched.
2047        if matches!(path.parent().map(|p| p.try_exists()), Some(Ok(true))) {
2048            watch_path!(watcher, &path, watch_path,);
2049        }
2050    }
2051}
2052
2053/// Try to read the blinded identity key for a given `TimePeriod`.
2054///
2055/// Returns `None` if the service is running in "offline" mode.
2056///
2057// TODO (#1194): we don't currently have support for "offline" mode so this can never return
2058// `Ok(None)`.
2059pub(super) fn read_blind_id_keypair(
2060    keymgr: &Arc<KeyMgr>,
2061    nickname: &HsNickname,
2062    period: TimePeriod,
2063) -> Result<Option<HsBlindIdKeypair>, FatalError> {
2064    let svc_key_spec = HsIdKeypairSpecifier::new(nickname.clone());
2065    let hsid_kp = keymgr
2066        .get::<HsIdKeypair>(&svc_key_spec)?
2067        .ok_or_else(|| FatalError::MissingHsIdKeypair(nickname.clone()))?;
2068
2069    let blind_id_key_spec = BlindIdKeypairSpecifier::new(nickname.clone(), period);
2070
2071    // TODO: make the keystore selector configurable
2072    let keystore_selector = Default::default();
2073    match keymgr.get::<HsBlindIdKeypair>(&blind_id_key_spec)? {
2074        Some(kp) => Ok(Some(kp)),
2075        None => {
2076            let (_hs_blind_id_key, hs_blind_id_kp, _subcredential) = hsid_kp
2077                .compute_blinded_key(period)
2078                .map_err(|_| internal!("failed to compute blinded key"))?;
2079
2080            // Note: we can't use KeyMgr::generate because this key is derived from the HsId
2081            // (KeyMgr::generate uses the tor_keymgr::Keygen trait under the hood,
2082            // which assumes keys are randomly generated, rather than derived from existing keys).
2083
2084            keymgr.insert(hs_blind_id_kp, &blind_id_key_spec, keystore_selector, true)?;
2085
2086            let arti_path = |spec: &dyn KeySpecifier| {
2087                spec.arti_path()
2088                    .map_err(into_internal!("invalid key specifier?!"))
2089            };
2090
2091            Ok(Some(
2092                keymgr.get::<HsBlindIdKeypair>(&blind_id_key_spec)?.ok_or(
2093                    FatalError::KeystoreRace {
2094                        action: "read",
2095                        path: arti_path(&blind_id_key_spec)?,
2096                    },
2097                )?,
2098            ))
2099        }
2100    }
2101}
2102
2103/// Determine the [`State`] of the publisher based on the upload results
2104/// from the current `time_periods`.
2105fn upload_result_state(
2106    netdir: &NetDir,
2107    time_periods: &[TimePeriodContext],
2108) -> (State, Option<Problem>) {
2109    let current_period = netdir.hs_time_period();
2110    let current_period_res = time_periods
2111        .iter()
2112        .find(|ctx| ctx.params.time_period() == current_period);
2113
2114    let succeeded_current_tp = current_period_res
2115        .iter()
2116        .flat_map(|res| &res.upload_results)
2117        .filter(|res| res.upload_res.is_ok())
2118        .collect_vec();
2119
2120    let secondary_tp_res = time_periods
2121        .iter()
2122        .filter(|ctx| ctx.params.time_period() != current_period)
2123        .collect_vec();
2124
2125    let succeeded_secondary_tp = secondary_tp_res
2126        .iter()
2127        .flat_map(|res| &res.upload_results)
2128        .filter(|res| res.upload_res.is_ok())
2129        .collect_vec();
2130
2131    // All of the failed uploads (for all TPs)
2132    let failed = time_periods
2133        .iter()
2134        .flat_map(|res| &res.upload_results)
2135        .filter(|res| res.upload_res.is_err())
2136        .collect_vec();
2137    let problems: Vec<DescUploadRetryError> = failed
2138        .iter()
2139        .flat_map(|e| e.upload_res.as_ref().map_err(|e| e.clone()).err())
2140        .collect();
2141
2142    let err = match problems.as_slice() {
2143        [_, ..] => Some(problems.into()),
2144        [] => None,
2145    };
2146
2147    if time_periods.len() < 2 {
2148        // We need at least TP contexts (one for the primary TP,
2149        // and another for the secondary one).
2150        //
2151        // If either is missing, we are unreachable for some or all clients.
2152        return (State::DegradedUnreachable, err);
2153    }
2154
2155    let state = match (
2156        succeeded_current_tp.as_slice(),
2157        succeeded_secondary_tp.as_slice(),
2158    ) {
2159        (&[], &[..]) | (&[..], &[]) if failed.is_empty() => {
2160            // We don't have any upload results for one or both TPs.
2161            // We are still bootstrapping.
2162            State::Bootstrapping
2163        }
2164        (&[_, ..], &[_, ..]) if failed.is_empty() => {
2165            // We have uploaded the descriptor to one or more HsDirs from both
2166            // HsDir rings (primary and secondary), and none of the uploads failed.
2167            // We are fully reachable.
2168            State::Running
2169        }
2170        (&[_, ..], &[_, ..]) => {
2171            // We have uploaded the descriptor to one or more HsDirs from both
2172            // HsDir rings (primary and secondary), but some of the uploads failed.
2173            // We are reachable, but we failed to upload the descriptor to all the HsDirs
2174            // that were supposed to have it.
2175            State::DegradedReachable
2176        }
2177        (&[..], &[]) | (&[], &[..]) => {
2178            // We have either
2179            //   * uploaded the descriptor to some of the HsDirs from one of the rings,
2180            //   but haven't managed to upload it to any of the HsDirs on the other ring, or
2181            //   * all of the uploads failed
2182            //
2183            // Either way, we are definitely not reachable by all clients.
2184            State::DegradedUnreachable
2185        }
2186    };
2187
2188    (state, err)
2189}
2190
2191/// Whether the reactor should initiate an upload.
2192#[derive(Copy, Clone, Debug, Default, PartialEq)]
2193enum PublishStatus {
2194    /// We need to call upload_all.
2195    UploadScheduled,
2196    /// We are rate-limited until the specified [`Instant`].
2197    ///
2198    /// We have tried to schedule multiple uploads in a short time span,
2199    /// and we are rate-limited. We are waiting for a signal from the schedule_upload_tx
2200    /// channel to unblock us.
2201    RateLimited(Instant),
2202    /// We are idle and waiting for external events.
2203    ///
2204    /// We have enough information to build the descriptor, but since we have already called
2205    /// upload_all to upload it to all relevant HSDirs, there is nothing for us to do right nbow.
2206    Idle,
2207    /// We are waiting for the IPT manager to establish some introduction points.
2208    ///
2209    /// No descriptors will be published until the `PublishStatus` of the reactor is changed to
2210    /// `UploadScheduled`.
2211    #[default]
2212    AwaitingIpts,
2213}
2214
2215/// The backoff schedule for the task that publishes descriptors.
2216#[derive(Clone, Debug)]
2217struct PublisherBackoffSchedule<M: Mockable> {
2218    /// The delays
2219    retry_delay: RetryDelay,
2220    /// The mockable reactor state, needed for obtaining an rng.
2221    mockable: M,
2222}
2223
2224impl<M: Mockable> BackoffSchedule for PublisherBackoffSchedule<M> {
2225    fn max_retries(&self) -> Option<usize> {
2226        None
2227    }
2228
2229    fn overall_timeout(&self) -> Option<Duration> {
2230        Some(OVERALL_UPLOAD_TIMEOUT)
2231    }
2232
2233    fn single_attempt_timeout(&self) -> Option<Duration> {
2234        Some(self.mockable.estimate_upload_timeout())
2235    }
2236
2237    fn next_delay<E: RetriableError>(&mut self, _error: &E) -> Option<Duration> {
2238        Some(self.retry_delay.next_delay(&mut self.mockable.thread_rng()))
2239    }
2240}
2241
2242impl RetriableError for UploadError {
2243    fn should_retry(&self) -> bool {
2244        match self {
2245            UploadError::Request(_) | UploadError::Circuit(_) | UploadError::Stream(_) => true,
2246            UploadError::Bug(_) => false,
2247        }
2248    }
2249}
2250
2251/// The outcome of uploading a descriptor to the HSDirs from a particular time period.
2252#[derive(Debug, Clone)]
2253struct TimePeriodUploadResult {
2254    /// The time period.
2255    time_period: TimePeriod,
2256    /// The upload results.
2257    hsdir_result: Vec<HsDirUploadStatus>,
2258}
2259
2260/// The outcome of uploading a descriptor to a particular HsDir.
2261#[derive(Clone, Debug)]
2262struct HsDirUploadStatus {
2263    /// The identity of the HsDir we attempted to upload the descriptor to.
2264    relay_ids: RelayIds,
2265    /// The outcome of this attempt.
2266    upload_res: UploadResult,
2267    /// The revision counter of the descriptor we tried to upload.
2268    revision_counter: RevisionCounter,
2269}
2270
2271/// The outcome of uploading a descriptor.
2272type UploadResult = Result<(), DescUploadRetryError>;
2273
2274impl From<BackoffError<UploadError>> for DescUploadRetryError {
2275    fn from(e: BackoffError<UploadError>) -> Self {
2276        use BackoffError as BE;
2277        use DescUploadRetryError as DURE;
2278
2279        match e {
2280            BE::FatalError(e) => DURE::FatalError(e),
2281            BE::MaxRetryCountExceeded(e) => DURE::MaxRetryCountExceeded(e),
2282            BE::Timeout(e) => DURE::Timeout(e),
2283            BE::ExplicitStop(_) => {
2284                DURE::Bug(internal!("explicit stop in publisher backoff schedule?!"))
2285            }
2286        }
2287    }
2288}
2289
2290// NOTE: the rest of the publisher tests live in publish.rs
2291#[cfg(test)]
2292mod test {
2293    // @@ begin test lint list maintained by maint/add_warning @@
2294    #![allow(clippy::bool_assert_comparison)]
2295    #![allow(clippy::clone_on_copy)]
2296    #![allow(clippy::dbg_macro)]
2297    #![allow(clippy::mixed_attributes_style)]
2298    #![allow(clippy::print_stderr)]
2299    #![allow(clippy::print_stdout)]
2300    #![allow(clippy::single_char_pattern)]
2301    #![allow(clippy::unwrap_used)]
2302    #![allow(clippy::unchecked_time_subtraction)]
2303    #![allow(clippy::useless_vec)]
2304    #![allow(clippy::needless_pass_by_value)]
2305    #![allow(clippy::string_slice)] // See arti#2571
2306    //! <!-- @@ end test lint list maintained by maint/add_warning @@ -->
2307    use super::*;
2308    use tor_netdir::testnet;
2309
2310    /// Create a `TimePeriodContext` from the specified upload results.
2311    fn create_time_period_ctx(
2312        params: &HsDirParams,
2313        upload_results: Vec<HsDirUploadStatus>,
2314    ) -> TimePeriodContext {
2315        TimePeriodContext {
2316            params: params.clone(),
2317            hs_dirs: vec![],
2318            last_successful: None,
2319            upload_results,
2320        }
2321    }
2322
2323    /// Create a single `HsDirUploadStatus`
2324    fn create_upload_status(upload_res: UploadResult) -> HsDirUploadStatus {
2325        HsDirUploadStatus {
2326            relay_ids: RelayIds::empty(),
2327            upload_res,
2328            revision_counter: RevisionCounter::from(13),
2329        }
2330    }
2331
2332    /// Create a bunch of results, all with the specified `upload_res`.
2333    fn create_upload_results(upload_res: UploadResult) -> Vec<HsDirUploadStatus> {
2334        std::iter::repeat_with(|| create_upload_status(upload_res.clone()))
2335            .take(10)
2336            .collect()
2337    }
2338
2339    fn construct_netdir() -> NetDir {
2340        const SRV1: [u8; 32] = *b"The door refused to open.       ";
2341        const SRV2: [u8; 32] = *b"It said, 'Five cents, please.'  ";
2342
2343        let dir = testnet::construct_custom_netdir(|_, _, bld| {
2344            bld.shared_rand_cur(7, SRV1.into(), None)
2345                .shared_rand_prev(7, SRV2.into(), None);
2346        })
2347        .unwrap();
2348
2349        dir.unwrap_if_sufficient().unwrap()
2350    }
2351
2352    #[test]
2353    fn upload_result_status_bootstrapping() {
2354        let netdir = construct_netdir();
2355        let all_params = netdir.hs_all_time_periods();
2356        let current_period = netdir.hs_time_period();
2357        let primary_params = all_params
2358            .iter()
2359            .find(|param| param.time_period() == current_period)
2360            .unwrap();
2361        let results = [
2362            (vec![], vec![]),
2363            (vec![], create_upload_results(Ok(()))),
2364            (create_upload_results(Ok(())), vec![]),
2365        ];
2366
2367        for (primary_result, secondary_result) in results {
2368            let primary_ctx = create_time_period_ctx(primary_params, primary_result);
2369
2370            let secondary_params = all_params
2371                .iter()
2372                .find(|param| param.time_period() != current_period)
2373                .unwrap();
2374            let secondary_ctx = create_time_period_ctx(secondary_params, secondary_result.clone());
2375
2376            let (status, err) = upload_result_state(&netdir, &[primary_ctx, secondary_ctx]);
2377            assert_eq!(status, State::Bootstrapping);
2378            assert!(err.is_none());
2379        }
2380    }
2381
2382    #[test]
2383    fn upload_result_status_running() {
2384        let netdir = construct_netdir();
2385        let all_params = netdir.hs_all_time_periods();
2386        let current_period = netdir.hs_time_period();
2387        let primary_params = all_params
2388            .iter()
2389            .find(|param| param.time_period() == current_period)
2390            .unwrap();
2391
2392        let secondary_result = create_upload_results(Ok(()));
2393        let secondary_params = all_params
2394            .iter()
2395            .find(|param| param.time_period() != current_period)
2396            .unwrap();
2397        let secondary_ctx = create_time_period_ctx(secondary_params, secondary_result.clone());
2398
2399        let primary_result = create_upload_results(Ok(()));
2400        let primary_ctx = create_time_period_ctx(primary_params, primary_result);
2401        let (status, err) = upload_result_state(&netdir, &[primary_ctx, secondary_ctx]);
2402        assert_eq!(status, State::Running);
2403        assert!(err.is_none());
2404    }
2405
2406    #[test]
2407    fn upload_result_status_reachable() {
2408        let netdir = construct_netdir();
2409        let all_params = netdir.hs_all_time_periods();
2410        let current_period = netdir.hs_time_period();
2411        let primary_params = all_params
2412            .iter()
2413            .find(|param| param.time_period() == current_period)
2414            .unwrap();
2415
2416        let primary_result = create_upload_results(Ok(()));
2417        let primary_ctx = create_time_period_ctx(primary_params, primary_result.clone());
2418        let failed_res = create_upload_results(Err(DescUploadRetryError::Bug(internal!("test"))));
2419        let secondary_result = create_upload_results(Ok(()))
2420            .into_iter()
2421            .chain(failed_res.iter().cloned())
2422            .collect();
2423        let secondary_params = all_params
2424            .iter()
2425            .find(|param| param.time_period() != current_period)
2426            .unwrap();
2427        let secondary_ctx = create_time_period_ctx(secondary_params, secondary_result);
2428        let (status, err) = upload_result_state(&netdir, &[primary_ctx, secondary_ctx]);
2429
2430        // Degraded but reachable (because some of the secondary HsDir uploads failed).
2431        assert_eq!(status, State::DegradedReachable);
2432        assert!(matches!(err, Some(Problem::DescriptorUpload(_))));
2433    }
2434
2435    #[test]
2436    fn upload_result_status_unreachable() {
2437        let netdir = construct_netdir();
2438        let all_params = netdir.hs_all_time_periods();
2439        let current_period = netdir.hs_time_period();
2440        let primary_params = all_params
2441            .iter()
2442            .find(|param| param.time_period() == current_period)
2443            .unwrap();
2444        let mut primary_result =
2445            create_upload_results(Err(DescUploadRetryError::Bug(internal!("test"))));
2446        let primary_ctx = create_time_period_ctx(primary_params, primary_result.clone());
2447        // No secondary TP (we are unreachable).
2448        let (status, err) = upload_result_state(&netdir, &[primary_ctx]);
2449        assert_eq!(status, State::DegradedUnreachable);
2450        assert!(matches!(err, Some(Problem::DescriptorUpload(_))));
2451
2452        // Add a successful result
2453        primary_result.push(create_upload_status(Ok(())));
2454        let primary_ctx = create_time_period_ctx(primary_params, primary_result.clone());
2455        let (status, err) = upload_result_state(&netdir, &[primary_ctx]);
2456        // Still degraded, and unreachable (because we don't have a TimePeriodContext
2457        // for the secondary TP)
2458        assert_eq!(status, State::DegradedUnreachable);
2459        assert!(matches!(err, Some(Problem::DescriptorUpload(_))));
2460
2461        // If we add another time period where none of the uploads were successful,
2462        // we're *still* unreachable
2463        let secondary_result =
2464            create_upload_results(Err(DescUploadRetryError::Bug(internal!("test"))));
2465        let secondary_params = all_params
2466            .iter()
2467            .find(|param| param.time_period() != current_period)
2468            .unwrap();
2469        let secondary_ctx = create_time_period_ctx(secondary_params, secondary_result.clone());
2470        let primary_ctx = create_time_period_ctx(primary_params, primary_result.clone());
2471        let (status, err) = upload_result_state(&netdir, &[primary_ctx, secondary_ctx]);
2472        assert_eq!(status, State::DegradedUnreachable);
2473        assert!(matches!(err, Some(Problem::DescriptorUpload(_))));
2474    }
2475}