diff --git a/cSpell.json b/cSpell.json index d8dee5c6b..88794b2ad 100644 --- a/cSpell.json +++ b/cSpell.json @@ -51,6 +51,7 @@ "proot", "Quickstart", "Rasterbar", + "reannounce", "repr", "reqwest", "rngs", diff --git a/packages/configuration/src/lib.rs b/packages/configuration/src/lib.rs index d42c82df9..8b4d9363d 100644 --- a/packages/configuration/src/lib.rs +++ b/packages/configuration/src/lib.rs @@ -59,17 +59,26 @@ impl HttpApi { pub struct Configuration { pub log_level: Option, pub mode: TrackerMode, + + // Database configuration pub db_driver: DatabaseDriver, pub db_path: String, + + /// Interval in seconds that the client should wait between sending regular announce requests to the tracker pub announce_interval: u32, + /// Minimum announce interval. Clients must not reannounce more frequently than this pub min_announce_interval: u32, - pub max_peer_timeout: u32, pub on_reverse_proxy: bool, pub external_ip: Option, pub tracker_usage_statistics: bool, pub persistent_torrent_completed_stat: bool, + + // Cleanup job configuration + pub max_peer_timeout: u32, pub inactive_peer_cleanup_interval: u64, pub remove_peerless_torrents: bool, + + // Server jobs configuration pub udp_trackers: Vec, pub http_trackers: Vec, pub http_api: HttpApi, diff --git a/src/bootstrap/app.rs b/src/bootstrap/app.rs index e845feac0..c0e688a0d 100644 --- a/src/bootstrap/app.rs +++ b/src/bootstrap/app.rs @@ -1,3 +1,16 @@ +//! Setup for the main tracker application. +//! +//! The [`setup`](bootstrap::app::setup) only builds the application and its dependencies but it does not start the application. +//! In fact, there is no such thing as the main application process. When the application starts, the only thing it does is +//! starting a bunch of independent jobs. If you are looking for how things are started you should read [`app::start`](crate::app::start) +//! function documentation. +//! +//! Setup steps: +//! +//! 1. Load the global application configuration. +//! 2. Initialize static variables. +//! 3. Initialize logging. +//! 4. Initialize the domain tracker. use std::env; use std::sync::Arc; @@ -9,6 +22,7 @@ use crate::shared::crypto::ephemeral_instance_keys; use crate::tracker::services::tracker_factory; use crate::tracker::Tracker; +/// It loads the configuration from the environment and builds the main domain [`tracker`](crate::tracker::Tracker) struct. #[must_use] pub fn setup() -> (Arc, Arc) { let configuration = Arc::new(initialize_configuration()); @@ -17,6 +31,9 @@ pub fn setup() -> (Arc, Arc) { (configuration, tracker) } +/// It initializes the application with the given configuration. +/// +/// The configuration may be obtained from the environment (via config file or env vars). #[must_use] pub fn initialize_with_configuration(configuration: &Arc) -> Arc { initialize_static(); @@ -24,6 +41,12 @@ pub fn initialize_with_configuration(configuration: &Arc) -> Arc< Arc::new(initialize_tracker(configuration)) } +/// It initializes the application static values. +/// +/// These values are accessible throughout the entire application: +/// +/// - The time when the application started. +/// - An ephemeral instance random seed. This seed is used for encryption and it's changed when the main application process is restarted. pub fn initialize_static() { // Set the time of Torrust app starting lazy_static::initialize(&static_time::TIME_AT_APP_START); @@ -32,6 +55,17 @@ pub fn initialize_static() { lazy_static::initialize(&ephemeral_instance_keys::RANDOM_SEED); } +/// It loads the application configuration from the environment. +/// +/// There are two methods to inject the configuration: +/// +/// 1. By using a config file: `config.toml`. The file must be in the same folder where you are running the tracker. +/// 2. Environment variable: `TORRUST_TRACKER_CONFIG`. The variable contains the same contents as the `config.toml` file. +/// +/// Environment variable has priority over the config file. +/// +/// Refer to the [configuration documentation](https://docs.rs/torrust-tracker-configuration) for the configuration options. +/// /// # Panics /// /// Will panic if it can't load the configuration from either @@ -50,11 +84,18 @@ fn initialize_configuration() -> Configuration { } } +/// It builds the domain tracker +/// +/// The tracker is the domain layer service. It's the entrypoint to make requests to the domain layer. +/// It's used by other higher-level components like the UDP and HTTP trackers or the tracker API. #[must_use] pub fn initialize_tracker(config: &Arc) -> Tracker { tracker_factory(config.clone()) } +/// It initializes the log level, format and channel. +/// +/// See [the logging setup](crate::bootstrap::logging::setup) for more info about logging. pub fn initialize_logging(config: &Arc) { bootstrap::logging::setup(config); } diff --git a/src/bootstrap/jobs/http_tracker.rs b/src/bootstrap/jobs/http_tracker.rs index 43bd0076f..ac0161640 100644 --- a/src/bootstrap/jobs/http_tracker.rs +++ b/src/bootstrap/jobs/http_tracker.rs @@ -1,3 +1,16 @@ +//! HTTP tracker job starter. +//! +//! The function [`http_tracker::start_job`](crate::bootstrap::jobs::http_tracker::start_job) starts a new HTTP tracker server. +//! +//! > **NOTICE**: the application can launch more than one HTTP tracker on different ports. +//! Refer to the [configuration documentation](https://docs.rs/torrust-tracker-configuration) for the configuration options. +//! +//! The [`http_tracker::start_job`](crate::bootstrap::jobs::http_tracker::start_job) function spawns a new asynchronous task, +//! that tasks is the "**launcher**". The "**launcher**" starts the actual server and sends a message back to the main application. +//! The main application waits until receives the message [`ServerJobStarted`](crate::bootstrap::jobs::http_tracker::ServerJobStarted) from the "**launcher**". +//! +//! The "**launcher**" is an intermediary thread that decouples the HTTP servers from the process that handles it. The HTTP could be used independently in the future. +//! In that case it would not need to notify a parent process. use std::sync::Arc; use axum_server::tls_rustls::RustlsConfig; @@ -10,9 +23,16 @@ use crate::servers::http::v1::launcher; use crate::servers::http::Version; use crate::tracker; +/// This is the message that the "**launcher**" spawned task sends to the main application process to notify that the HTTP server was successfully started. +/// +/// > **NOTICE**: it does not mean the HTTP server is ready to receive requests. It only means the new server started. It might take some time to the server to be ready to accept request. #[derive(Debug)] pub struct ServerJobStarted(); +/// It starts a new HTTP server with the provided configuration and version. +/// +/// Right now there is only one version but in the future we could support more than one HTTP tracker version at the same time. +/// This feature allows supporting breaking changes on `BitTorrent` BEPs. pub async fn start_job(config: &HttpTracker, tracker: Arc, version: Version) -> JoinHandle<()> { match version { Version::V1 => start_v1(config, tracker.clone()).await, diff --git a/src/bootstrap/jobs/mod.rs b/src/bootstrap/jobs/mod.rs index ba44a56ad..c519a9f4b 100644 --- a/src/bootstrap/jobs/mod.rs +++ b/src/bootstrap/jobs/mod.rs @@ -1,3 +1,11 @@ +//! Application jobs launchers. +//! +//! The main application setup has only two main stages: +//! +//! 1. Setup the domain layer: the core tracker. +//! 2. Launch all the application services as concurrent jobs. +//! +//! This modules contains all the functions needed to start those jobs. pub mod http_tracker; pub mod torrent_cleanup; pub mod tracker_apis; diff --git a/src/bootstrap/jobs/torrent_cleanup.rs b/src/bootstrap/jobs/torrent_cleanup.rs index 64240bffe..d48769139 100644 --- a/src/bootstrap/jobs/torrent_cleanup.rs +++ b/src/bootstrap/jobs/torrent_cleanup.rs @@ -1,3 +1,15 @@ +//! Job that runs a task on intervals to clean up torrents. +//! +//! It removes inactive peers and (optionally) peerless torrents. +//! +//! **Inactive peers** are peers that have not been updated for more than `max_peer_timeout` seconds. +//! `max_peer_timeout` is a customizable core tracker option. +//! +//! If the core tracker configuration option `remove_peerless_torrents` is true, the cleanup job will also +//! remove **peerless torrents** which are torrents with an empty peer list. +//! +//! Refer to [`torrust-tracker-configuration documentation`](https://docs.rs/torrust-tracker-configuration) for more info about those options. + use std::sync::Arc; use chrono::Utc; @@ -7,6 +19,11 @@ use torrust_tracker_configuration::Configuration; use crate::tracker; +/// It starts a jobs for cleaning up the torrent data in the tracker. +/// +/// The cleaning task is executed on an `inactive_peer_cleanup_interval`. +/// +/// Refer to [`torrust-tracker-configuration documentation`](https://docs.rs/torrust-tracker-configuration) for more info about that option. #[must_use] pub fn start_job(config: &Arc, tracker: &Arc) -> JoinHandle<()> { let weak_tracker = std::sync::Arc::downgrade(tracker); diff --git a/src/bootstrap/jobs/tracker_apis.rs b/src/bootstrap/jobs/tracker_apis.rs index cdebc21a8..9afe4ab24 100644 --- a/src/bootstrap/jobs/tracker_apis.rs +++ b/src/bootstrap/jobs/tracker_apis.rs @@ -1,3 +1,25 @@ +//! Tracker API job starter. +//! +//! The [`tracker_apis::start_job`](crate::bootstrap::jobs::tracker_apis::start_job) +//! function starts a the HTTP tracker REST API. +//! +//! > **NOTICE**: that even thought there is only one job the API has different +//! versions. API consumers can choose which version to use. The API version is +//! part of the URL, for example: `http://localhost:1212/api/v1/stats`. +//! +//! The [`tracker_apis::start_job`](crate::bootstrap::jobs::tracker_apis::start_job) +//! function spawns a new asynchronous task, that tasks is the "**launcher**". +//! The "**launcher**" starts the actual server and sends a message back +//! to the main application. The main application waits until receives +//! the message [`ApiServerJobStarted`](crate::bootstrap::jobs::tracker_apis::ApiServerJobStarted) +//! from the "**launcher**". +//! +//! The "**launcher**" is an intermediary thread that decouples the API server +//! from the process that handles it. The API could be used independently +//! in the future. In that case it would not need to notify a parent process. +//! +//! Refer to the [configuration documentation](https://docs.rs/torrust-tracker-configuration) +//! for the API configuration options. use std::sync::Arc; use axum_server::tls_rustls::RustlsConfig; @@ -9,9 +31,21 @@ use torrust_tracker_configuration::HttpApi; use crate::servers::apis::server; use crate::tracker; +/// This is the message that the "launcher" spawned task sends to the main +/// application process to notify the API server was successfully started. +/// +/// > **NOTICE**: it does not mean the API server is ready to receive requests. +/// It only means the new server started. It might take some time to the server +/// to be ready to accept request. #[derive(Debug)] pub struct ApiServerJobStarted(); +/// This function starts a new API server with the provided configuration. +/// +/// The functions starts a new concurrent task that will run the API server. +/// This task will send a message to the main application process to notify +/// that the API server was successfully started. +/// /// # Panics /// /// It would panic if unable to send the `ApiServerJobStarted` notice. diff --git a/src/bootstrap/jobs/udp_tracker.rs b/src/bootstrap/jobs/udp_tracker.rs index 138222daf..76c465a8d 100644 --- a/src/bootstrap/jobs/udp_tracker.rs +++ b/src/bootstrap/jobs/udp_tracker.rs @@ -1,3 +1,11 @@ +//! UDP tracker job starter. +//! +//! The [`udp_tracker::start_job`](crate::bootstrap::jobs::udp_tracker::start_job) +//! function starts a new UDP tracker server. +//! +//! > **NOTICE**: that the application can launch more than one UDP tracker +//! on different ports. Refer to the [configuration documentation](https://docs.rs/torrust-tracker-configuration) +//! for the configuration options. use std::sync::Arc; use log::{error, info, warn}; @@ -7,6 +15,9 @@ use torrust_tracker_configuration::UdpTracker; use crate::servers::udp::server::Udp; use crate::tracker; +/// It starts a new UDP server with the provided configuration. +/// +/// It spawns a new asynchronous task for the new UDP server. #[must_use] pub fn start_job(config: &UdpTracker, tracker: Arc) -> JoinHandle<()> { let bind_addr = config.bind_address.clone(); diff --git a/src/bootstrap/logging.rs b/src/bootstrap/logging.rs index 83e2c9360..97e26919d 100644 --- a/src/bootstrap/logging.rs +++ b/src/bootstrap/logging.rs @@ -1,3 +1,15 @@ +//! Setup for the application logging. +//! +//! It redirects the log info to the standard output with the log level defined in the configuration. +//! +//! - `Off` +//! - `Error` +//! - `Warn` +//! - `Info` +//! - `Debug` +//! - `Trace` +//! +//! Refer to the [configuration crate documentation](https://docs.rs/torrust-tracker-configuration) to know how to change log settings. use std::str::FromStr; use std::sync::Once; @@ -6,6 +18,7 @@ use torrust_tracker_configuration::Configuration; static INIT: Once = Once::new(); +/// It redirects the log info to the standard output with the log level defined in the configuration pub fn setup(cfg: &Configuration) { let level = config_level_or_default(&cfg.log_level); diff --git a/src/bootstrap/mod.rs b/src/bootstrap/mod.rs index e3b6467ee..e39cf3adc 100644 --- a/src/bootstrap/mod.rs +++ b/src/bootstrap/mod.rs @@ -1,3 +1,10 @@ +//! Tracker application bootstrapping. +//! +//! This module includes all the functions to build the application, its dependencies, and run the jobs. +//! +//! Jobs are tasks executed concurrently. Some of them are concurrent because of the asynchronous nature of the task, +//! like cleaning torrents, and other jobs because they can be enabled/disabled depending on the configuration. +//! For example, you can have more than one UDP and HTTP tracker, each server is executed like a independent job. pub mod app; pub mod jobs; pub mod logging; diff --git a/src/lib.rs b/src/lib.rs index 3e7225923..a460a28b8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,10 +27,15 @@ //! - [Features](#features) //! - [Services](#services) //! - [Installation](#installation) +//! - [Minimum requirements](#minimum-requirements) +//! - [Prerequisites](#prerequisites) +//! - [Install from sources](#install-from-sources) +//! - [Run with docker](#run-with-docker) //! - [Configuration](#configuration) //! - [Usage](#usage) //! - [Components](#components) //! - [Implemented BEPs](#implemented-beps) +//! - [Contributing](#contributing) //! //! # Features //! @@ -356,7 +361,7 @@ //! - The [`UDP`](crate::servers::udp) tracker //! - The [`HTTP`](crate::servers::http) tracker //! -//! ![Torrust Tracker Components](https://github.com/torrust/torrust-tracker/blob/main/docs/media/torrust-tracker-components.png) +//! ![Torrust Tracker Components](https://raw.githubusercontent.com/torrust/torrust-tracker/main/docs/media/torrust-tracker-components.png) //! //! ## Core tracker //! @@ -421,6 +426,10 @@ //! - [BEP 27](https://www.bittorrent.org/beps/bep_0027.html): Private Torrents //! - [BEP 41](https://www.bittorrent.org/beps/bep_0041.html): UDP Tracker Protocol Extensions //! - [BEP 48](https://www.bittorrent.org/beps/bep_0048.html): Tracker Protocol Extension: Scrape +//! +//! # Contributing +//! +//! If you want to contribute to this documentation you can [open a new pull request](https://github.com/torrust/torrust-tracker/pulls). pub mod app; pub mod bootstrap; pub mod servers; diff --git a/src/tracker/auth.rs b/src/tracker/auth.rs index 31e1f50e4..9068a94f0 100644 --- a/src/tracker/auth.rs +++ b/src/tracker/auth.rs @@ -1,3 +1,38 @@ +//! Tracker authentication services and structs. +//! +//! This module contains functions to handle tracker keys. +//! Tracker keys are tokens used to authenticate the tracker clients when the tracker runs +//! in `private` or `private_listed` modes. +//! +//! There are services to [`generate`](crate::tracker::auth::generate) and [`verify`](crate::tracker::auth::verify) authentication keys. +//! +//! Authentication keys are used only by [`HTTP`](crate::servers::http) trackers. All keys have an expiration time, that means +//! they are only valid during a period of time. After that time the expiring key will no longer be valid. +//! +//! Keys are stored in this struct: +//! +//! ```rust,no_run +//! pub struct ExpiringKey { +//! /// Random 32-char string. For example: `YZSl4lMZupRuOpSRC3krIKR5BPB14nrJ` +//! pub key: Key, +//! /// Timestamp, the key will be no longer valid after this timestamp +//! pub valid_until: DurationSinceUnixEpoch, +//! } +//! ``` +//! +//! You can generate a new key valid for `9999` seconds and `0` nanoseconds from the current time with the following: +//! +//! ```rust,no_run +//! let expiring_key = auth::generate(Duration::new(9999, 0)); +//! +//! assert!(auth::verify(&expiring_key).is_ok()); +//! ``` +//! +//! And you can later verify it with: +//! +//! ```rust,no_run +//! assert!(auth::verify(&expiring_key).is_ok()); +//! ``` use std::panic::Location; use std::str::FromStr; use std::sync::Arc; @@ -15,6 +50,8 @@ use crate::shared::bit_torrent::common::AUTH_KEY_LENGTH; use crate::shared::clock::{convert_from_timestamp_to_datetime_utc, Current, DurationSinceUnixEpoch, Time, TimeNow}; #[must_use] +/// It generates a new random 32-char authentication [`ExpiringKey`](crate::tracker::auth::ExpiringKey) +/// /// # Panics /// /// It would panic if the `lifetime: Duration` + Duration is more than `Duration::MAX`. @@ -33,6 +70,8 @@ pub fn generate(lifetime: Duration) -> ExpiringKey { } } +/// It verifies an [`ExpiringKey`](crate::tracker::auth::ExpiringKey). It checks if the expiration date has passed. +/// /// # Errors /// /// Will return `Error::KeyExpired` if `auth_key.valid_until` is past the `current_time`. @@ -50,9 +89,13 @@ pub fn verify(auth_key: &ExpiringKey) -> Result<(), Error> { } } +/// An authentication key which has an expiration time. +/// After that time is will automatically become invalid. #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Clone)] pub struct ExpiringKey { + /// Random 32-char string. For example: `YZSl4lMZupRuOpSRC3krIKR5BPB14nrJ` pub key: Key, + /// Timestamp, the key will be no longer valid after this timestamp pub valid_until: DurationSinceUnixEpoch, } @@ -82,9 +125,24 @@ impl ExpiringKey { } } +/// A randomly generated token used for authentication. +/// +/// It contains lower and uppercase letters and numbers. +/// It's a 32-char string. #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Clone, Display, Hash)] pub struct Key(String); +/// Error returned when a key cannot be parsed from a string. +/// +/// ```rust,no_run +/// let key_string = "YZSl4lMZupRuOpSRC3krIKR5BPB14nrJ"; +/// let key = Key::from_str(key_string); +/// +/// assert!(key.is_ok()); +/// assert_eq!(key.unwrap().to_string(), key_string); +/// ``` +/// +/// If the string does not contains a valid key, the parser function will return this error. #[derive(Debug, PartialEq, Eq)] pub struct ParseKeyError; @@ -100,6 +158,8 @@ impl FromStr for Key { } } +/// Verification error. Error returned when an [`ExpiringKey`](crate::tracker::auth::ExpiringKey) cannot be verified with the [`verify(...)`](crate::tracker::auth::verify) function. +/// #[derive(Debug, Error)] #[allow(dead_code)] pub enum Error { diff --git a/src/tracker/databases/driver.rs b/src/tracker/databases/driver.rs index 4ce6ea515..ef9a4eb07 100644 --- a/src/tracker/databases/driver.rs +++ b/src/tracker/databases/driver.rs @@ -1,3 +1,7 @@ +//! Database driver factory. +//! +//! See [`databases::driver::build`](crate::tracker::databases::driver::build) +//! function for more information. use torrust_tracker_primitives::DatabaseDriver; use super::error::Error; @@ -5,7 +9,28 @@ use super::mysql::Mysql; use super::sqlite::Sqlite; use super::{Builder, Database}; -/// . +/// It builds a new database driver. +/// +/// Example for `SQLite3`: +/// +/// ```rust,no_run +/// let db_driver = "Sqlite3".to_string(); +/// let db_path = "./storage/database/data.db".to_string(); +/// let database = databases::driver::build(&db_driver, &db_path)?; +/// ``` +/// +/// Example for `MySQL`: +/// +/// ```rust,no_run +/// let db_driver = "MySQL".to_string(); +/// let db_path = "mysql://db_user:db_user_secret_password@mysql:3306/torrust_tracker".to_string(); +/// let database = databases::driver::build(&db_driver, &db_path)?; +/// ``` +/// +/// Refer to the [configuration documentation](https://docs.rs/torrust-tracker-configuration) +/// for more information about the database configuration. +/// +/// > **WARNING**: The driver instantiation runs database migrations. /// /// # Errors /// diff --git a/src/tracker/databases/error.rs b/src/tracker/databases/error.rs index 68b732190..d89ec05de 100644 --- a/src/tracker/databases/error.rs +++ b/src/tracker/databases/error.rs @@ -1,3 +1,6 @@ +//! Database errors. +//! +//! This module contains the [Database errors](crate::tracker::databases::error::Error). use std::panic::Location; use std::sync::Arc; @@ -7,24 +10,28 @@ use torrust_tracker_primitives::DatabaseDriver; #[derive(thiserror::Error, Debug, Clone)] pub enum Error { + /// The query unexpectedly returned nothing. #[error("The {driver} query unexpectedly returned nothing: {source}")] QueryReturnedNoRows { source: LocatedError<'static, dyn std::error::Error + Send + Sync>, driver: DatabaseDriver, }, + /// The query was malformed. #[error("The {driver} query was malformed: {source}")] InvalidQuery { source: LocatedError<'static, dyn std::error::Error + Send + Sync>, driver: DatabaseDriver, }, + /// Unable to insert a record into the database #[error("Unable to insert record into {driver} database, {location}")] InsertFailed { location: &'static Location<'static>, driver: DatabaseDriver, }, + /// Unable to delete a record into the database #[error("Failed to remove record from {driver} database, error-code: {error_code}, {location}")] DeleteFailed { location: &'static Location<'static>, @@ -32,12 +39,14 @@ pub enum Error { driver: DatabaseDriver, }, + /// Unable to connect to the database #[error("Failed to connect to {driver} database: {source}")] ConnectionError { source: LocatedError<'static, UrlError>, driver: DatabaseDriver, }, + /// Unable to create a connection pool #[error("Failed to create r2d2 {driver} connection pool: {source}")] ConnectionPool { source: LocatedError<'static, r2d2::Error>, diff --git a/src/tracker/databases/mod.rs b/src/tracker/databases/mod.rs index f68288bbe..3b02415df 100644 --- a/src/tracker/databases/mod.rs +++ b/src/tracker/databases/mod.rs @@ -1,3 +1,48 @@ +//! The persistence module. +//! +//! Persistence is currently implemented with one [`Database`](crate::tracker::databases::Database) trait. +//! +//! There are two implementations of the trait (two drivers): +//! +//! - [`Mysql`](crate::tracker::databases::mysql::Mysql) +//! - [`Sqlite`](crate::tracker::databases::sqlite::Sqlite) +//! +//! > **NOTICE**: There are no database migrations. If there are any changes, +//! we will implemented them or provide a script to migrate to the new schema. +//! +//! The persistent objects are: +//! +//! - [Torrent metrics](#torrent-metrics) +//! - [Torrent whitelist](torrent-whitelist) +//! - [Authentication keys](authentication-keys) +//! +//! # Torrent metrics +//! +//! Field | Sample data | Description +//! ---|---|--- +//! `id` | 1 | Autoincrement id +//! `info_hash` | `c1277613db1d28709b034a017ab2cae4be07ae10` | `BitTorrent` infohash V1 +//! `completed` | 20 | The number of peers that have ever completed downloading the torrent associated to this entry. See [`Entry`](crate::tracker::torrent::Entry) for more information. +//! +//! > **NOTICE**: The peer list for a torrent is not persisted. Since peer have to re-announce themselves on intervals, the data is be +//! regenerated again after some minutes. +//! +//! # Torrent whitelist +//! +//! Field | Sample data | Description +//! ---|---|--- +//! `id` | 1 | Autoincrement id +//! `info_hash` | `c1277613db1d28709b034a017ab2cae4be07ae10` | `BitTorrent` infohash V1 +//! +//! # Authentication keys +//! +//! Field | Sample data | Description +//! ---|---|--- +//! `id` | 1 | Autoincrement id +//! `key` | `IrweYtVuQPGbG9Jzx1DihcPmJGGpVy82` | Token +//! `valid_until` | 1672419840 | Timestamp for the expiring date +//! +//! > **NOTICE**: All keys must have an expiration date. pub mod driver; pub mod error; pub mod mysql; @@ -32,9 +77,10 @@ where } } +/// The persistence trait. It contains all the methods to interact with the database. #[async_trait] pub trait Database: Sync + Send { - /// . + /// It instantiates a new database driver. /// /// # Errors /// @@ -43,39 +89,142 @@ pub trait Database: Sync + Send { where Self: std::marker::Sized; - /// . + // Schema + + /// It generates the database tables. SQL queries are hardcoded in the trait + /// implementation. + /// + /// # Context: Schema /// /// # Errors /// /// Will return `Error` if unable to create own tables. fn create_database_tables(&self) -> Result<(), Error>; + /// It drops the database tables. + /// + /// # Context: Schema + /// /// # Errors /// /// Will return `Err` if unable to drop tables. fn drop_database_tables(&self) -> Result<(), Error>; + // Torrent Metrics + + /// It loads the torrent metrics data from the database. + /// + /// It returns an array of tuples with the torrent + /// [`InfoHash`](crate::shared::bit_torrent::info_hash::InfoHash) and the + /// [`completed`](crate::tracker::torrent::Entry::completed) counter + /// which is the number of times the torrent has been downloaded. + /// See [`Entry::completed`](crate::tracker::torrent::Entry::completed). + /// + /// # Context: Torrent Metrics + /// + /// # Errors + /// + /// Will return `Err` if unable to load. async fn load_persistent_torrents(&self) -> Result, Error>; - async fn load_keys(&self) -> Result, Error>; + /// It saves the torrent metrics data into the database. + /// + /// # Context: Torrent Metrics + /// + /// # Errors + /// + /// Will return `Err` if unable to save. + async fn save_persistent_torrent(&self, info_hash: &InfoHash, completed: u32) -> Result<(), Error>; - async fn load_whitelist(&self) -> Result, Error>; + // Whitelist - async fn save_persistent_torrent(&self, info_hash: &InfoHash, completed: u32) -> Result<(), Error>; + /// It loads the whitelisted torrents from the database. + /// + /// # Context: Whitelist + /// + /// # Errors + /// + /// Will return `Err` if unable to load. + async fn load_whitelist(&self) -> Result, Error>; + /// It checks if the torrent is whitelisted. + /// + /// It returns `Some(InfoHash)` if the torrent is whitelisted, `None` otherwise. + /// + /// # Context: Whitelist + /// + /// # Errors + /// + /// Will return `Err` if unable to load. async fn get_info_hash_from_whitelist(&self, info_hash: &InfoHash) -> Result, Error>; + /// It adds the torrent to the whitelist. + /// + /// # Context: Whitelist + /// + /// # Errors + /// + /// Will return `Err` if unable to save. async fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result; + /// It checks if the torrent is whitelisted. + /// + /// # Context: Whitelist + /// + /// # Errors + /// + /// Will return `Err` if unable to load. + async fn is_info_hash_whitelisted(&self, info_hash: &InfoHash) -> Result { + Ok(self.get_info_hash_from_whitelist(info_hash).await?.is_some()) + } + + /// It removes the torrent from the whitelist. + /// + /// # Context: Whitelist + /// + /// # Errors + /// + /// Will return `Err` if unable to save. async fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result; + // Authentication keys + + /// It loads the expiring authentication keys from the database. + /// + /// # Context: Authentication Keys + /// + /// # Errors + /// + /// Will return `Err` if unable to load. + async fn load_keys(&self) -> Result, Error>; + + /// It gets an expiring authentication key from the database. + /// + /// It returns `Some(ExpiringKey)` if a [`ExpiringKey`](crate::tracker::auth::ExpiringKey) + /// with the input [`Key`](crate::tracker::auth::Key) exists, `None` otherwise. + /// + /// # Context: Authentication Keys + /// + /// # Errors + /// + /// Will return `Err` if unable to load. async fn get_key_from_keys(&self, key: &Key) -> Result, Error>; + /// It adds an expiring authentication key to the database. + /// + /// # Context: Authentication Keys + /// + /// # Errors + /// + /// Will return `Err` if unable to save. async fn add_key_to_keys(&self, auth_key: &auth::ExpiringKey) -> Result; + /// It removes an expiring authentication key from the database. + /// + /// # Context: Authentication Keys + /// + /// # Errors + /// + /// Will return `Err` if unable to load. async fn remove_key_from_keys(&self, key: &Key) -> Result; - - async fn is_info_hash_whitelisted(&self, info_hash: &InfoHash) -> Result { - Ok(self.get_info_hash_from_whitelist(info_hash).await?.is_some()) - } } diff --git a/src/tracker/databases/mysql.rs b/src/tracker/databases/mysql.rs index 7e4aab99e..4419666ab 100644 --- a/src/tracker/databases/mysql.rs +++ b/src/tracker/databases/mysql.rs @@ -1,3 +1,4 @@ +//! The `MySQL` database driver. use std::str::FromStr; use std::time::Duration; @@ -22,6 +23,10 @@ pub struct Mysql { #[async_trait] impl Database for Mysql { + /// It instantiates a new `MySQL` database driver. + /// + /// Refer to [`databases::Database::new`](crate::tracker::databases::Database::new). + /// /// # Errors /// /// Will return `r2d2::Error` if `db_path` is not able to create `MySQL` database. @@ -34,6 +39,7 @@ impl Database for Mysql { Ok(Self { pool }) } + /// Refer to [`databases::Database::create_database_tables`](crate::tracker::databases::Database::create_database_tables). fn create_database_tables(&self) -> Result<(), Error> { let create_whitelist_table = " CREATE TABLE IF NOT EXISTS whitelist ( @@ -73,6 +79,7 @@ impl Database for Mysql { Ok(()) } + /// Refer to [`databases::Database::drop_database_tables`](crate::tracker::databases::Database::drop_database_tables). fn drop_database_tables(&self) -> Result<(), Error> { let drop_whitelist_table = " DROP TABLE `whitelist`;" @@ -97,6 +104,7 @@ impl Database for Mysql { Ok(()) } + /// Refer to [`databases::Database::load_persistent_torrents`](crate::tracker::databases::Database::load_persistent_torrents). async fn load_persistent_torrents(&self) -> Result, Error> { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; @@ -111,6 +119,7 @@ impl Database for Mysql { Ok(torrents) } + /// Refer to [`databases::Database::load_keys`](crate::tracker::databases::Database::load_keys). async fn load_keys(&self) -> Result, Error> { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; @@ -125,6 +134,7 @@ impl Database for Mysql { Ok(keys) } + /// Refer to [`databases::Database::load_whitelist`](crate::tracker::databases::Database::load_whitelist). async fn load_whitelist(&self) -> Result, Error> { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; @@ -135,6 +145,7 @@ impl Database for Mysql { Ok(info_hashes) } + /// Refer to [`databases::Database::save_persistent_torrent`](crate::tracker::databases::Database::save_persistent_torrent). async fn save_persistent_torrent(&self, info_hash: &InfoHash, completed: u32) -> Result<(), Error> { const COMMAND : &str = "INSERT INTO torrents (info_hash, completed) VALUES (:info_hash_str, :completed) ON DUPLICATE KEY UPDATE completed = VALUES(completed)"; @@ -147,6 +158,7 @@ impl Database for Mysql { Ok(conn.exec_drop(COMMAND, params! { info_hash_str, completed })?) } + /// Refer to [`databases::Database::get_info_hash_from_whitelist`](crate::tracker::databases::Database::get_info_hash_from_whitelist). async fn get_info_hash_from_whitelist(&self, info_hash: &InfoHash) -> Result, Error> { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; @@ -160,6 +172,7 @@ impl Database for Mysql { Ok(info_hash) } + /// Refer to [`databases::Database::add_info_hash_to_whitelist`](crate::tracker::databases::Database::add_info_hash_to_whitelist). async fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; @@ -173,6 +186,7 @@ impl Database for Mysql { Ok(1) } + /// Refer to [`databases::Database::remove_info_hash_from_whitelist`](crate::tracker::databases::Database::remove_info_hash_from_whitelist). async fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; @@ -183,6 +197,7 @@ impl Database for Mysql { Ok(1) } + /// Refer to [`databases::Database::get_key_from_keys`](crate::tracker::databases::Database::get_key_from_keys). async fn get_key_from_keys(&self, key: &Key) -> Result, Error> { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; @@ -199,6 +214,7 @@ impl Database for Mysql { })) } + /// Refer to [`databases::Database::add_key_to_keys`](crate::tracker::databases::Database::add_key_to_keys). async fn add_key_to_keys(&self, auth_key: &auth::ExpiringKey) -> Result { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; @@ -213,6 +229,7 @@ impl Database for Mysql { Ok(1) } + /// Refer to [`databases::Database::remove_key_from_keys`](crate::tracker::databases::Database::remove_key_from_keys). async fn remove_key_from_keys(&self, key: &Key) -> Result { let mut conn = self.pool.get().map_err(|e| (e, DRIVER))?; diff --git a/src/tracker/databases/sqlite.rs b/src/tracker/databases/sqlite.rs index 931289183..1968ee049 100644 --- a/src/tracker/databases/sqlite.rs +++ b/src/tracker/databases/sqlite.rs @@ -1,3 +1,4 @@ +//! The `SQLite3` database driver. use std::panic::Location; use std::str::FromStr; @@ -19,6 +20,10 @@ pub struct Sqlite { #[async_trait] impl Database for Sqlite { + /// It instantiates a new `SQLite3` database driver. + /// + /// Refer to [`databases::Database::new`](crate::tracker::databases::Database::new). + /// /// # Errors /// /// Will return `r2d2::Error` if `db_path` is not able to create `SqLite` database. @@ -27,6 +32,7 @@ impl Database for Sqlite { Pool::new(cm).map_or_else(|err| Err((err, DatabaseDriver::Sqlite3).into()), |pool| Ok(Sqlite { pool })) } + /// Refer to [`databases::Database::create_database_tables`](crate::tracker::databases::Database::create_database_tables). fn create_database_tables(&self) -> Result<(), Error> { let create_whitelist_table = " CREATE TABLE IF NOT EXISTS whitelist ( @@ -60,6 +66,7 @@ impl Database for Sqlite { Ok(()) } + /// Refer to [`databases::Database::drop_database_tables`](crate::tracker::databases::Database::drop_database_tables). fn drop_database_tables(&self) -> Result<(), Error> { let drop_whitelist_table = " DROP TABLE whitelist;" @@ -82,6 +89,7 @@ impl Database for Sqlite { Ok(()) } + /// Refer to [`databases::Database::load_persistent_torrents`](crate::tracker::databases::Database::load_persistent_torrents). async fn load_persistent_torrents(&self) -> Result, Error> { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; @@ -102,6 +110,7 @@ impl Database for Sqlite { Ok(torrents) } + /// Refer to [`databases::Database::load_keys`](crate::tracker::databases::Database::load_keys). async fn load_keys(&self) -> Result, Error> { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; @@ -122,6 +131,7 @@ impl Database for Sqlite { Ok(keys) } + /// Refer to [`databases::Database::load_whitelist`](crate::tracker::databases::Database::load_whitelist). async fn load_whitelist(&self) -> Result, Error> { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; @@ -138,6 +148,7 @@ impl Database for Sqlite { Ok(info_hashes) } + /// Refer to [`databases::Database::save_persistent_torrent`](crate::tracker::databases::Database::save_persistent_torrent). async fn save_persistent_torrent(&self, info_hash: &InfoHash, completed: u32) -> Result<(), Error> { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; @@ -156,6 +167,7 @@ impl Database for Sqlite { } } + /// Refer to [`databases::Database::get_info_hash_from_whitelist`](crate::tracker::databases::Database::get_info_hash_from_whitelist). async fn get_info_hash_from_whitelist(&self, info_hash: &InfoHash) -> Result, Error> { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; @@ -168,6 +180,7 @@ impl Database for Sqlite { Ok(query.map(|f| InfoHash::from_str(&f.get_unwrap::<_, String>(0)).unwrap())) } + /// Refer to [`databases::Database::add_info_hash_to_whitelist`](crate::tracker::databases::Database::add_info_hash_to_whitelist). async fn add_info_hash_to_whitelist(&self, info_hash: InfoHash) -> Result { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; @@ -183,6 +196,7 @@ impl Database for Sqlite { } } + /// Refer to [`databases::Database::remove_info_hash_from_whitelist`](crate::tracker::databases::Database::remove_info_hash_from_whitelist). async fn remove_info_hash_from_whitelist(&self, info_hash: InfoHash) -> Result { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; @@ -200,6 +214,7 @@ impl Database for Sqlite { } } + /// Refer to [`databases::Database::get_key_from_keys`](crate::tracker::databases::Database::get_key_from_keys). async fn get_key_from_keys(&self, key: &Key) -> Result, Error> { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; @@ -219,6 +234,7 @@ impl Database for Sqlite { })) } + /// Refer to [`databases::Database::add_key_to_keys`](crate::tracker::databases::Database::add_key_to_keys). async fn add_key_to_keys(&self, auth_key: &auth::ExpiringKey) -> Result { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; @@ -237,6 +253,7 @@ impl Database for Sqlite { } } + /// Refer to [`databases::Database::remove_key_from_keys`](crate::tracker::databases::Database::remove_key_from_keys). async fn remove_key_from_keys(&self, key: &Key) -> Result { let conn = self.pool.get().map_err(|e| (e, DRIVER))?; diff --git a/src/tracker/error.rs b/src/tracker/error.rs index aaf755e0d..f1e622673 100644 --- a/src/tracker/error.rs +++ b/src/tracker/error.rs @@ -1,7 +1,16 @@ +//! Error returned by the core `Tracker`. +//! +//! Error | Context | Description +//! ---|---|--- +//! `PeerKeyNotValid` | Authentication | The supplied key is not valid. It may not be registered or expired. +//! `PeerNotAuthenticated` | Authentication | The peer did not provide the authentication key. +//! `TorrentNotWhitelisted` | Authorization | The action cannot be perform on a not-whitelisted torrent (it only applies for trackers running in `listed` or `private_listed` modes). +//! use std::panic::Location; use torrust_tracker_located_error::LocatedError; +/// Authentication or authorization error returned by the core `Tracker` #[derive(thiserror::Error, Debug, Clone)] pub enum Error { // Authentication errors diff --git a/src/tracker/mod.rs b/src/tracker/mod.rs index a89d6df2c..ce69b6125 100644 --- a/src/tracker/mod.rs +++ b/src/tracker/mod.rs @@ -1,3 +1,412 @@ +//! The core `tracker` module contains the generic `BitTorrent` tracker logic which is independent of the delivery layer. +//! +//! It contains the tracker services and their dependencies. It's a domain layer which does not +//! specify how the end user should connect to the `Tracker`. +//! +//! Typically this module is intended to be used by higher modules like: +//! +//! - A UDP tracker +//! - A HTTP tracker +//! - A tracker REST API +//! +//! ```text +//! Delivery layer Domain layer +//! +//! HTTP tracker | +//! UDP tracker |> Core tracker +//! Tracker REST API | +//! ``` +//! +//! # Table of contents +//! +//! - [Tracker](#tracker) +//! - [Announce request](#announce-request) +//! - [Scrape request](#scrape-request) +//! - [Torrents](#torrents) +//! - [Peers](#peers) +//! - [Configuration](#configuration) +//! - [Services](#services) +//! - [Authentication](#authentication) +//! - [Statistics](#statistics) +//! - [Persistence](#persistence) +//! +//! # Tracker +//! +//! The `Tracker` is the main struct in this module. `The` tracker has some groups of responsibilities: +//! +//! - **Core tracker**: it handles the information about torrents and peers. +//! - **Authentication**: it handles authentication keys which are used by HTTP trackers. +//! - **Authorization**: it handles the permission to perform requests. +//! - **Whitelist**: when the tracker runs in `listed` or `private_listed` mode all operations are restricted to whitelisted torrents. +//! - **Statistics**: it keeps and serves the tracker statistics. +//! +//! Refer to [torrust-tracker-configuration](https://docs.rs/torrust-tracker-configuration) crate docs to get more information about the tracker settings. +//! +//! ## Announce request +//! +//! Handling `announce` requests is the most important task for a `BitTorrent` tracker. +//! +//! A `BitTorrent` swarm is a network of peers that are all trying to download the same torrent. +//! When a peer wants to find other peers it announces itself to the swarm via the tracker. +//! The peer sends its data to the tracker so that the tracker can add it to the swarm. +//! The tracker responds to the peer with the list of other peers in the swarm so that +//! the peer can contact them to start downloading pieces of the file from them. +//! +//! Once you have instantiated the `Tracker` you can `announce` a new [`peer`](crate::tracker::peer::Peer) with: +//! +//! ```rust,no_run +//! let info_hash = InfoHash { +//! "3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0".parse::().unwrap() +//! }; +//! +//! let peer = Peer { +//! peer_id: peer::Id(*b"-qB00000000000000001"), +//! peer_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(126, 0, 0, 1)), 8081), +//! updated: DurationSinceUnixEpoch::new(1_669_397_478_934, 0), +//! uploaded: NumberOfBytes(0), +//! downloaded: NumberOfBytes(0), +//! left: NumberOfBytes(0), +//! event: AnnounceEvent::Completed, +//! } +//! +//! let peer_ip = IpAddr::V4(Ipv4Addr::from_str("126.0.0.1").unwrap()); +//! +//! let announce_data = tracker.announce(&info_hash, &mut peer, &peer_ip).await; +//! ``` +//! +//! The `Tracker` returns the list of peers for the torrent with the infohash `3b245504cf5f11bbdbe1201cea6a6bf45aee1bc0`, +//! filtering out the peer that is making the `announce` request. +//! +//! > **NOTICE**: that the peer argument is mutable because the `Tracker` can change the peer IP if the peer is using a loopback IP. +//! +//! The `peer_ip` argument is the resolved peer ip. It's a common practice that trackers ignore the peer ip in the `announce` request params, +//! and resolve the peer ip using the IP of the client making the request. As the tracker is a domain service, the peer IP must be provided +//! for the `Tracker` user, which is usually a higher component with access the the request metadata, for example, connection data, proxy headers, +//! etcetera. +//! +//! The returned struct is: +//! +//! ```rust,no_run +//! pub struct AnnounceData { +//! pub peers: Vec, +//! pub swarm_stats: SwarmStats, +//! pub interval: u32, // Option `announce_interval` from core tracker configuration +//! pub interval_min: u32, // Option `min_announce_interval` from core tracker configuration +//! } +//! +//! pub struct SwarmStats { +//! pub completed: u32, // The number of peers that have ever completed downloading +//! pub seeders: u32, // The number of active peers that have completed downloading (seeders) +//! pub leechers: u32, // The number of active peers that have not completed downloading (leechers) +//! } +//! +//! // Core tracker configuration +//! pub struct Configuration { +//! // ... +//! pub announce_interval: u32, // Interval in seconds that the client should wait between sending regular announce requests to the tracker +//! pub min_announce_interval: u32, // Minimum announce interval. Clients must not reannounce more frequently than this +//! // ... +//! } +//! ``` +//! +//! Refer to `BitTorrent` BEPs and other sites for more information about the `announce` request: +//! +//! - [BEP 3. The `BitTorrent` Protocol Specification](https://www.bittorrent.org/beps/bep_0003.html) +//! - [BEP 23. Tracker Returns Compact Peer Lists](https://www.bittorrent.org/beps/bep_0023.html) +//! - [Vuze docs](https://wiki.vuze.com/w/Announce) +//! +//! ## Scrape request +//! +//! The `scrape` request allows clients to query metadata about the swarm in bulk. +//! +//! An `scrape` request includes a list of infohashes whose swarm metadata you want to collect. +//! +//! The returned struct is: +//! +//! ```rust,no_run +//! pub struct ScrapeData { +//! pub files: HashMap, +//! } +//! +//! pub struct SwarmMetadata { +//! pub complete: u32, // The number of active peers that have completed downloading (seeders) +//! pub downloaded: u32, // The number of peers that have ever completed downloading +//! pub incomplete: u32, // The number of active peers that have not completed downloading (leechers) +//! } +//! ``` +//! +//! The JSON representation of a sample `scrape` response would be like the following: +//! +//! ```json +//! { +//! 'files': { +//! 'xxxxxxxxxxxxxxxxxxxx': {'complete': 11, 'downloaded': 13772, 'incomplete': 19}, +//! 'yyyyyyyyyyyyyyyyyyyy': {'complete': 21, 'downloaded': 206, 'incomplete': 20} +//! } +//! } +//! ``` +//! +//! `xxxxxxxxxxxxxxxxxxxx` and `yyyyyyyyyyyyyyyyyyyy` are 20-byte infohash arrays. +//! There are two data structures for infohashes: byte arrays and hex strings: +//! +//! ```rust,no_run +//! let info_hash: InfoHash = [255u8; 20].into(); +//! +//! assert_eq!( +//! info_hash, +//! InfoHash::from_str("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF").unwrap() +//! ); +//! ``` +//! Refer to `BitTorrent` BEPs and other sites for more information about the `scrape` request: +//! +//! - [BEP 48. Tracker Protocol Extension: Scrape](https://www.bittorrent.org/beps/bep_0048.html) +//! - [BEP 15. UDP Tracker Protocol for `BitTorrent`. Scrape section](https://www.bittorrent.org/beps/bep_0015.html) +//! - [Vuze docs](https://wiki.vuze.com/w/Scrape) +//! +//! ## Torrents +//! +//! The [`torrent`](crate::tracker::torrent) module contains all the data structures stored by the `Tracker` except for peers. +//! +//! We can represent the data stored in memory internally by the `Tracker` with this JSON object: +//! +//! ```json +//! { +//! "c1277613db1d28709b034a017ab2cae4be07ae10": { +//! "completed": 0, +//! "peers": { +//! "-qB00000000000000001": { +//! "peer_id": "-qB00000000000000001", +//! "peer_addr": "2.137.87.41:1754", +//! "updated": 1672419840, +//! "uploaded": 120, +//! "downloaded": 60, +//! "left": 60, +//! "event": "started" +//! }, +//! "-qB00000000000000002": { +//! "peer_id": "-qB00000000000000002", +//! "peer_addr": "23.17.287.141:2345", +//! "updated": 1679415984, +//! "uploaded": 80, +//! "downloaded": 20, +//! "left": 40, +//! "event": "started" +//! } +//! } +//! } +//! } +//! ``` +//! +//! The `Tracker` maintains an indexed-by-info-hash list of torrents. For each torrent, it stores a torrent `Entry`. +//! The torrent entry has two attributes: +//! +//! - `completed`: which is hte number of peers that have completed downloading the torrent file/s. As they have completed downloading, +//! they have a full version of the torrent data, and they can provide the full data to other peers. That's why they are also known as "seeders". +//! - `peers`: an indexed and orderer list of peer for the torrent. Each peer contains the data received from the peer in the `announce` request. +//! +//! The [`torrent`](crate::tracker::torrent) module not only contains the original data obtained from peer via `announce` requests, it also contains +//! aggregate data that can be derived from the original data. For example: +//! +//! ```rust,no_run +//! pub struct SwarmMetadata { +//! pub complete: u32, // The number of active peers that have completed downloading (seeders) +//! pub downloaded: u32, // The number of peers that have ever completed downloading +//! pub incomplete: u32, // The number of active peers that have not completed downloading (leechers) +//! } +//! +//! pub struct SwarmStats { +//! pub completed: u32, // The number of peers that have ever completed downloading +//! pub seeders: u32, // The number of active peers that have completed downloading (seeders) +//! pub leechers: u32, // The number of active peers that have not completed downloading (leechers) +//! } +//! ``` +//! +//! > **NOTICE**: that `complete` or `completed` peers are the peers that have completed downloading, but only the active ones are considered "seeders". +//! +//! `SwarmStats` struct follows name conventions for `scrape` responses. See [BEP 48](https://www.bittorrent.org/beps/bep_0048.html), while `SwarmStats` +//! is used for the rest of cases. +//! +//! Refer to [`torrent`](crate::tracker::torrent) module for more details about these data structures. +//! +//! ## Peers +//! +//! A `Peer` is the struct used by the `Tracker` to keep peers data: +//! +//! ```rust,no_run +//! pub struct Peer { +//! pub peer_id: Id, // The peer ID +//! pub peer_addr: SocketAddr, // Peer socket address +//! pub updated: DurationSinceUnixEpoch, // Last time (timestamp) when the peer was updated +//! pub uploaded: NumberOfBytes, // Number of bytes the peer has uploaded so far +//! pub downloaded: NumberOfBytes, // Number of bytes the peer has downloaded so far +//! pub left: NumberOfBytes, // The number of bytes this peer still has to download +//! pub event: AnnounceEvent, // The event the peer has announced: `started`, `completed`, `stopped` +//! } +//! ``` +//! +//! Notice that most of the attributes are obtained from the `announce` request. +//! For example, an HTTP announce request would contain the following `GET` parameters: +//! +//! +//! +//! The `Tracker` keeps an in-memory ordered data structure with all the torrents and a list of peers for each torrent, together with some swarm metrics. +//! +//! We can represent the data stored in memory with this JSON object: +//! +//! ```json +//! { +//! "c1277613db1d28709b034a017ab2cae4be07ae10": { +//! "completed": 0, +//! "peers": { +//! "-qB00000000000000001": { +//! "peer_id": "-qB00000000000000001", +//! "peer_addr": "2.137.87.41:1754", +//! "updated": 1672419840, +//! "uploaded": 120, +//! "downloaded": 60, +//! "left": 60, +//! "event": "started" +//! }, +//! "-qB00000000000000002": { +//! "peer_id": "-qB00000000000000002", +//! "peer_addr": "23.17.287.141:2345", +//! "updated": 1679415984, +//! "uploaded": 80, +//! "downloaded": 20, +//! "left": 40, +//! "event": "started" +//! } +//! } +//! } +//! } +//! ``` +//! +//! That JSON object does not exist, it's only a representation of the `Tracker` torrents data. +//! +//! `c1277613db1d28709b034a017ab2cae4be07ae10` is the torrent infohash and `completed` contains the number of peers +//! that have a full version of the torrent data, also known as seeders. +//! +//! Refer to [`peer`](crate::tracker::peer) module for more information about peers. +//! +//! # Configuration +//! +//! You can control the behavior of this module with the module settings: +//! +//! ```toml +//! log_level = "debug" +//! mode = "public" +//! db_driver = "Sqlite3" +//! db_path = "./storage/database/data.db" +//! announce_interval = 120 +//! min_announce_interval = 120 +//! max_peer_timeout = 900 +//! on_reverse_proxy = false +//! external_ip = "2.137.87.41" +//! tracker_usage_statistics = true +//! persistent_torrent_completed_stat = true +//! inactive_peer_cleanup_interval = 600 +//! remove_peerless_torrents = false +//! ``` +//! +//! Refer to the [`configuration` module documentation](https://docs.rs/torrust-tracker-configuration) to get more information about all options. +//! +//! # Services +//! +//! Services are domain services on top of the core tracker. Right now there are two types of service: +//! +//! - For statistics +//! - For torrents +//! +//! Services usually format the data inside the tracker to make it easier to consume by other parts. +//! They also decouple the internal data structure, used by the tracker, from the way we deliver that data to the consumers. +//! The internal data structure is designed for performance or low memory consumption. And it should be changed +//! without affecting the external consumers. +//! +//! Services can include extra features like pagination, for example. +//! +//! Refer to [`services`](crate::tracker::services) module for more information about services. +//! +//! # Authentication +//! +//! One of the core `Tracker` responsibilities is to create and keep authentication keys. Auth keys are used by HTTP trackers +//! when the tracker is running in `private` or `private_listed` mode. +//! +//! HTTP tracker's clients need to obtain an auth key before starting requesting the tracker. Once the get one they have to include +//! a `PATH` param with the key in all the HTTP requests. For example, when a peer wants to `announce` itself it has to use the +//! HTTP tracker endpoint `GET /announce/:key`. +//! +//! The common way to obtain the keys is by using the tracker API directly or via other applications like the [Torrust Index](https://github.com/torrust/torrust-index). +//! +//! To learn more about tracker authentication, refer to the following modules : +//! +//! - [`auth`](crate::tracker::auth) module. +//! - [`tracker`](crate::tracker) module. +//! - [`http`](crate::servers::http) module. +//! +//! # Statistics +//! +//! The `Tracker` keeps metrics for some events: +//! +//! ```rust,no_run +//! pub struct Metrics { +//! // IP version 4 +//! +//! // HTTP tracker +//! pub tcp4_connections_handled: u64, +//! pub tcp4_announces_handled: u64, +//! pub tcp4_scrapes_handled: u64, +//! +//! // UDP tracker +//! pub udp4_connections_handled: u64, +//! pub udp4_announces_handled: u64, +//! pub udp4_scrapes_handled: u64, +//! +//! // IP version 6 +//! +//! // HTTP tracker +//! pub tcp6_connections_handled: u64, +//! pub tcp6_announces_handled: u64, +//! pub tcp6_scrapes_handled: u64, +//! +//! // UDP tracker +//! pub udp6_connections_handled: u64, +//! pub udp6_announces_handled: u64, +//! pub udp6_scrapes_handled: u64, +//! } +//! ``` +//! +//! The metrics maintained by the `Tracker` are: +//! +//! - `connections_handled`: number of connections handled by the tracker +//! - `announces_handled`: number of `announce` requests handled by the tracker +//! - `scrapes_handled`: number of `scrape` handled requests by the tracker +//! +//! > **NOTICE**: as the HTTP tracker does not have an specific `connection` request like the UDP tracker, `connections_handled` are +//! increased on every `announce` and `scrape` requests. +//! +//! The tracker exposes an event sender API that allows the tracker users to send events. When a higher application service handles a +//! `connection` , `announce` or `scrape` requests, it notifies the `Tracker` by sending statistics events. +//! +//! For example, the HTTP tracker would send an event like the following when it handles an `announce` request received from a peer using IP version 4. +//! +//! ```rust,no_run +//! tracker.send_stats_event(statistics::Event::Tcp4Announce).await +//! ``` +//! +//! Refer to [`statistics`](crate::tracker::statistics) module for more information about statistics. +//! +//! # Persistence +//! +//! Right now the `Tracker` is responsible for storing and load data into and +//! from the database, when persistence is enabled. +//! +//! There are three types of persistent object: +//! +//! - Authentication keys (only expiring keys) +//! - Torrent whitelist +//! - Torrent metrics +//! +//! Refer to [`databases`](crate::tracker::databases) module for more information about persistence. pub mod auth; pub mod databases; pub mod error; @@ -25,28 +434,45 @@ use self::torrent::{SwarmMetadata, SwarmStats}; use crate::shared::bit_torrent::info_hash::InfoHash; use crate::tracker::databases::Database; +/// The domain layer tracker service. +/// +/// Its main responsibility is to handle the `announce` and `scrape` requests. +/// But it's also a container for the `Tracker` configuration, persistence, +/// authentication and other services. +/// +/// > **NOTICE**: the `Tracker` is not responsible for handling the network layer. +/// Typically, the `Tracker` is used by a higher application service that handles +/// the network layer. pub struct Tracker { + /// `Tracker` configuration. See pub config: Arc, + /// A database driver implementation: [`Sqlite3`](crate::tracker::databases::sqlite) + /// or [`MySQL`](crate::tracker::databases::mysql) + pub database: Box, mode: TrackerMode, keys: RwLock>, whitelist: RwLock>, torrents: RwLock>, stats_event_sender: Option>, stats_repository: statistics::Repo, - pub database: Box, } +/// Structure that holds general `Tracker` torrents metrics. +/// +/// Metrics are aggregate values for all torrents. #[derive(Debug, PartialEq, Default)] pub struct TorrentsMetrics { - // code-review: consider using `SwarmStats` for - // `seeders`, `completed`, and `leechers` attributes. - // pub swarm_stats: SwarmStats; + /// Total number of seeders for all torrents pub seeders: u64, + /// Total number of peers that have ever completed downloading for all torrents. pub completed: u64, + /// Total number of leechers for all torrents. pub leechers: u64, + /// Total number of torrents. pub torrents: u64, } +/// Structure that holds the data returned by the `announce` request. #[derive(Debug, PartialEq, Default)] pub struct AnnounceData { pub peers: Vec, @@ -55,6 +481,7 @@ pub struct AnnounceData { pub interval_min: u32, } +/// Structure that holds the data returned by the `scrape` request. #[derive(Debug, PartialEq, Default)] pub struct ScrapeData { pub files: HashMap, @@ -88,9 +515,11 @@ impl ScrapeData { } impl Tracker { + /// `Tracker` constructor. + /// /// # Errors /// - /// Will return a `databases::error::Error` if unable to connect to database. + /// Will return a `databases::error::Error` if unable to connect to database. The `Tracker` is responsible for the persistence. pub fn new( config: Arc, stats_event_sender: Option>, @@ -130,6 +559,8 @@ impl Tracker { /// It handles an announce request. /// + /// # Context: Tracker + /// /// BEP 03: [The `BitTorrent` Protocol Specification](https://www.bittorrent.org/beps/bep_0003.html). pub async fn announce(&self, info_hash: &InfoHash, peer: &mut Peer, remote_client_ip: &IpAddr) -> AnnounceData { // code-review: maybe instead of mutating the peer we could just return @@ -163,6 +594,8 @@ impl Tracker { /// It handles a scrape request. /// + /// # Context: Tracker + /// /// BEP 48: [Tracker Protocol Extension: Scrape](https://www.bittorrent.org/beps/bep_0048.html). pub async fn scrape(&self, info_hashes: &Vec) -> ScrapeData { let mut scrape_data = ScrapeData::empty(); @@ -178,6 +611,7 @@ impl Tracker { scrape_data } + /// It returns the data for a `scrape` response. async fn get_swarm_metadata(&self, info_hash: &InfoHash) -> SwarmMetadata { let torrents = self.get_torrents().await; match torrents.get(info_hash) { @@ -186,6 +620,168 @@ impl Tracker { } } + /// It loads the torrents from database into memory. It only loads the torrent entry list with the number of seeders for each torrent. + /// Peers data is not persisted. + /// + /// # Context: Tracker + /// + /// # Errors + /// + /// Will return a `database::Error` if unable to load the list of `persistent_torrents` from the database. + pub async fn load_torrents_from_database(&self) -> Result<(), databases::error::Error> { + let persistent_torrents = self.database.load_persistent_torrents().await?; + + let mut torrents = self.torrents.write().await; + + for (info_hash, completed) in persistent_torrents { + // Skip if torrent entry already exists + if torrents.contains_key(&info_hash) { + continue; + } + + let torrent_entry = torrent::Entry { + peers: BTreeMap::default(), + completed, + }; + + torrents.insert(info_hash, torrent_entry); + } + + Ok(()) + } + + async fn get_peers_for_peer(&self, info_hash: &InfoHash, peer: &Peer) -> Vec { + let read_lock = self.torrents.read().await; + + match read_lock.get(info_hash) { + None => vec![], + Some(entry) => entry.get_peers_for_peer(peer).into_iter().copied().collect(), + } + } + + /// # Context: Tracker + /// + /// Get all torrent peers for a given torrent + pub async fn get_all_torrent_peers(&self, info_hash: &InfoHash) -> Vec { + let read_lock = self.torrents.read().await; + + match read_lock.get(info_hash) { + None => vec![], + Some(entry) => entry.get_all_peers().into_iter().copied().collect(), + } + } + + /// It updates the torrent entry in memory, it also stores in the database + /// the torrent info data which is persistent, and finally return the data + /// needed for a `announce` request response. + /// + /// # Context: Tracker + pub async fn update_torrent_with_peer_and_get_stats(&self, info_hash: &InfoHash, peer: &peer::Peer) -> torrent::SwarmStats { + // code-review: consider splitting the function in two (command and query segregation). + // `update_torrent_with_peer` and `get_stats` + + let mut torrents = self.torrents.write().await; + + let torrent_entry = match torrents.entry(*info_hash) { + Entry::Vacant(vacant) => vacant.insert(torrent::Entry::new()), + Entry::Occupied(entry) => entry.into_mut(), + }; + + let stats_updated = torrent_entry.update_peer(peer); + + // todo: move this action to a separate worker + if self.config.persistent_torrent_completed_stat && stats_updated { + let _ = self + .database + .save_persistent_torrent(info_hash, torrent_entry.completed) + .await; + } + + let (seeders, completed, leechers) = torrent_entry.get_stats(); + + torrent::SwarmStats { + completed, + seeders, + leechers, + } + } + + pub async fn get_torrents(&self) -> RwLockReadGuard<'_, BTreeMap> { + self.torrents.read().await + } + + /// It calculates and returns the general `Tracker` + /// [`TorrentsMetrics`](crate::tracker::TorrentsMetrics) + /// + /// # Context: Tracker + pub async fn get_torrents_metrics(&self) -> TorrentsMetrics { + let mut torrents_metrics = TorrentsMetrics { + seeders: 0, + completed: 0, + leechers: 0, + torrents: 0, + }; + + let db = self.get_torrents().await; + + db.values().for_each(|torrent_entry| { + let (seeders, completed, leechers) = torrent_entry.get_stats(); + torrents_metrics.seeders += u64::from(seeders); + torrents_metrics.completed += u64::from(completed); + torrents_metrics.leechers += u64::from(leechers); + torrents_metrics.torrents += 1; + }); + + torrents_metrics + } + + /// Remove inactive peers and (optionally) peerless torrents + /// + /// # Context: Tracker + pub async fn cleanup_torrents(&self) { + let mut torrents_lock = self.torrents.write().await; + + // If we don't need to remove torrents we will use the faster iter + if self.config.remove_peerless_torrents { + torrents_lock.retain(|_, torrent_entry| { + torrent_entry.remove_inactive_peers(self.config.max_peer_timeout); + + if self.config.persistent_torrent_completed_stat { + torrent_entry.completed > 0 || !torrent_entry.peers.is_empty() + } else { + !torrent_entry.peers.is_empty() + } + }); + } else { + for (_, torrent_entry) in torrents_lock.iter_mut() { + torrent_entry.remove_inactive_peers(self.config.max_peer_timeout); + } + } + } + + /// It authenticates the peer `key` against the `Tracker` authentication + /// key list. + /// + /// # Errors + /// + /// Will return an error if the the authentication key cannot be verified. + /// + /// # Context: Authentication + pub async fn authenticate(&self, key: &Key) -> Result<(), auth::Error> { + if self.is_private() { + self.verify_auth_key(key).await + } else { + Ok(()) + } + } + + /// It generates a new expiring authentication key. + /// `lifetime` param is the duration in seconds for the new key. + /// The key will be no longer valid after `lifetime` seconds. + /// Authentication keys are used by HTTP trackers. + /// + /// # Context: Authentication + /// /// # Errors /// /// Will return a `database::Error` if unable to add the `auth_key` to the database. @@ -196,6 +792,10 @@ impl Tracker { Ok(auth_key) } + /// It removes an authentication key. + /// + /// # Context: Authentication + /// /// # Errors /// /// Will return a `database::Error` if unable to remove the `key` to the database. @@ -209,6 +809,10 @@ impl Tracker { Ok(()) } + /// It verifies an authentication key. + /// + /// # Context: Authentication + /// /// # Errors /// /// Will return a `key::Error` if unable to get any `auth_key`. @@ -224,6 +828,13 @@ impl Tracker { } } + /// The `Tracker` stores the authentication keys in memory and in the database. + /// In case you need to restart the `Tracker` you can load the keys from the database + /// into memory with this function. Keys are automatically stored in the database when they + /// are generated. + /// + /// # Context: Authentication + /// /// # Errors /// /// Will return a `database::Error` if unable to `load_keys` from the database. @@ -240,84 +851,10 @@ impl Tracker { Ok(()) } - /// Adding torrents is not relevant to public trackers. + /// It authenticates and authorizes a UDP tracker request. /// - /// # Errors + /// # Context: Authentication and Authorization /// - /// Will return a `database::Error` if unable to add the `info_hash` into the whitelist database. - pub async fn add_torrent_to_whitelist(&self, info_hash: &InfoHash) -> Result<(), databases::error::Error> { - self.add_torrent_to_database_whitelist(info_hash).await?; - self.add_torrent_to_memory_whitelist(info_hash).await; - Ok(()) - } - - /// It adds a torrent to the whitelist if it has not been whitelisted previously - async fn add_torrent_to_database_whitelist(&self, info_hash: &InfoHash) -> Result<(), databases::error::Error> { - let is_whitelisted = self.database.is_info_hash_whitelisted(info_hash).await?; - - if is_whitelisted { - return Ok(()); - } - - self.database.add_info_hash_to_whitelist(*info_hash).await?; - - Ok(()) - } - - pub async fn add_torrent_to_memory_whitelist(&self, info_hash: &InfoHash) -> bool { - self.whitelist.write().await.insert(*info_hash) - } - - /// Removing torrents is not relevant to public trackers. - /// - /// # Errors - /// - /// Will return a `database::Error` if unable to remove the `info_hash` from the whitelist database. - pub async fn remove_torrent_from_whitelist(&self, info_hash: &InfoHash) -> Result<(), databases::error::Error> { - self.remove_torrent_from_database_whitelist(info_hash).await?; - self.remove_torrent_from_memory_whitelist(info_hash).await; - Ok(()) - } - - /// # Errors - /// - /// Will return a `database::Error` if unable to remove the `info_hash` from the whitelist database. - pub async fn remove_torrent_from_database_whitelist(&self, info_hash: &InfoHash) -> Result<(), databases::error::Error> { - let is_whitelisted = self.database.is_info_hash_whitelisted(info_hash).await?; - - if !is_whitelisted { - return Ok(()); - } - - self.database.remove_info_hash_from_whitelist(*info_hash).await?; - - Ok(()) - } - - pub async fn remove_torrent_from_memory_whitelist(&self, info_hash: &InfoHash) -> bool { - self.whitelist.write().await.remove(info_hash) - } - - pub async fn is_info_hash_whitelisted(&self, info_hash: &InfoHash) -> bool { - self.whitelist.read().await.contains(info_hash) - } - - /// # Errors - /// - /// Will return a `database::Error` if unable to load the list whitelisted `info_hash`s from the database. - pub async fn load_whitelist_from_database(&self) -> Result<(), databases::error::Error> { - let whitelisted_torrents_from_database = self.database.load_whitelist().await?; - let mut whitelist = self.whitelist.write().await; - - whitelist.clear(); - - for info_hash in whitelisted_torrents_from_database { - let _ = whitelist.insert(info_hash); - } - - Ok(()) - } - /// # Errors /// /// Will return a `torrent::Error::PeerKeyNotValid` if the `key` is not valid. @@ -325,6 +862,7 @@ impl Tracker { /// Will return a `torrent::Error::PeerNotAuthenticated` if the `key` is `None`. /// /// Will return a `torrent::Error::TorrentNotWhitelisted` if the the Tracker is in listed mode and the `info_hash` is not whitelisted. + #[deprecated(since = "3.0.0", note = "please use `authenticate` and `authorize` instead")] pub async fn authenticate_request(&self, info_hash: &InfoHash, key: &Option) -> Result<(), Error> { // todo: this is a deprecated method. // We're splitting authentication and authorization responsibilities. @@ -369,18 +907,10 @@ impl Tracker { Ok(()) } - /// # Errors + /// Right now, there is only authorization when the `Tracker` runs in + /// `listed` or `private_listed` modes. /// - /// Will return an error if the the authentication key cannot be verified. - pub async fn authenticate(&self, key: &Key) -> Result<(), auth::Error> { - if self.is_private() { - self.verify_auth_key(key).await - } else { - Ok(()) - } - } - - /// The only authorization process is the whitelist. + /// # Context: Authorization /// /// # Errors /// @@ -401,139 +931,120 @@ impl Tracker { }); } - /// Loading the torrents from database into memory + /// It adds a torrent to the whitelist. + /// Adding torrents is not relevant to public trackers. + /// + /// # Context: Whitelist /// /// # Errors /// - /// Will return a `database::Error` if unable to load the list of `persistent_torrents` from the database. - pub async fn load_torrents_from_database(&self) -> Result<(), databases::error::Error> { - let persistent_torrents = self.database.load_persistent_torrents().await?; - - let mut torrents = self.torrents.write().await; - - for (info_hash, completed) in persistent_torrents { - // Skip if torrent entry already exists - if torrents.contains_key(&info_hash) { - continue; - } - - let torrent_entry = torrent::Entry { - peers: BTreeMap::default(), - completed, - }; - - torrents.insert(info_hash, torrent_entry); - } - + /// Will return a `database::Error` if unable to add the `info_hash` into the whitelist database. + pub async fn add_torrent_to_whitelist(&self, info_hash: &InfoHash) -> Result<(), databases::error::Error> { + self.add_torrent_to_database_whitelist(info_hash).await?; + self.add_torrent_to_memory_whitelist(info_hash).await; Ok(()) } - async fn get_peers_for_peer(&self, info_hash: &InfoHash, peer: &Peer) -> Vec { - let read_lock = self.torrents.read().await; + /// It adds a torrent to the whitelist if it has not been whitelisted previously + async fn add_torrent_to_database_whitelist(&self, info_hash: &InfoHash) -> Result<(), databases::error::Error> { + let is_whitelisted = self.database.is_info_hash_whitelisted(info_hash).await?; - match read_lock.get(info_hash) { - None => vec![], - Some(entry) => entry.get_peers_for_peer(peer).into_iter().copied().collect(), + if is_whitelisted { + return Ok(()); } - } - /// Get all torrent peers for a given torrent - pub async fn get_all_torrent_peers(&self, info_hash: &InfoHash) -> Vec { - let read_lock = self.torrents.read().await; + self.database.add_info_hash_to_whitelist(*info_hash).await?; - match read_lock.get(info_hash) { - None => vec![], - Some(entry) => entry.get_all_peers().into_iter().copied().collect(), - } + Ok(()) } - pub async fn update_torrent_with_peer_and_get_stats(&self, info_hash: &InfoHash, peer: &peer::Peer) -> torrent::SwarmStats { - // code-review: consider splitting the function in two (command and query segregation). - // `update_torrent_with_peer` and `get_stats` - - let mut torrents = self.torrents.write().await; + pub async fn add_torrent_to_memory_whitelist(&self, info_hash: &InfoHash) -> bool { + self.whitelist.write().await.insert(*info_hash) + } - let torrent_entry = match torrents.entry(*info_hash) { - Entry::Vacant(vacant) => vacant.insert(torrent::Entry::new()), - Entry::Occupied(entry) => entry.into_mut(), - }; + /// It removes a torrent from the whitelist. + /// Removing torrents is not relevant to public trackers. + /// + /// # Context: Whitelist + /// + /// # Errors + /// + /// Will return a `database::Error` if unable to remove the `info_hash` from the whitelist database. + pub async fn remove_torrent_from_whitelist(&self, info_hash: &InfoHash) -> Result<(), databases::error::Error> { + self.remove_torrent_from_database_whitelist(info_hash).await?; + self.remove_torrent_from_memory_whitelist(info_hash).await; + Ok(()) + } - let stats_updated = torrent_entry.update_peer(peer); + /// It removes a torrent from the whitelist in the database. + /// + /// # Context: Whitelist + /// + /// # Errors + /// + /// Will return a `database::Error` if unable to remove the `info_hash` from the whitelist database. + pub async fn remove_torrent_from_database_whitelist(&self, info_hash: &InfoHash) -> Result<(), databases::error::Error> { + let is_whitelisted = self.database.is_info_hash_whitelisted(info_hash).await?; - // todo: move this action to a separate worker - if self.config.persistent_torrent_completed_stat && stats_updated { - let _ = self - .database - .save_persistent_torrent(info_hash, torrent_entry.completed) - .await; + if !is_whitelisted { + return Ok(()); } - let (seeders, completed, leechers) = torrent_entry.get_stats(); + self.database.remove_info_hash_from_whitelist(*info_hash).await?; - torrent::SwarmStats { - completed, - seeders, - leechers, - } + Ok(()) } - pub async fn get_torrents(&self) -> RwLockReadGuard<'_, BTreeMap> { - self.torrents.read().await + /// It removes a torrent from the whitelist in memory. + /// + /// # Context: Whitelist + pub async fn remove_torrent_from_memory_whitelist(&self, info_hash: &InfoHash) -> bool { + self.whitelist.write().await.remove(info_hash) } - pub async fn get_torrents_metrics(&self) -> TorrentsMetrics { - let mut torrents_metrics = TorrentsMetrics { - seeders: 0, - completed: 0, - leechers: 0, - torrents: 0, - }; + /// It checks if a torrent is whitelisted. + /// + /// # Context: Whitelist + pub async fn is_info_hash_whitelisted(&self, info_hash: &InfoHash) -> bool { + self.whitelist.read().await.contains(info_hash) + } - let db = self.get_torrents().await; + /// It loads the whitelist from the database. + /// + /// # Context: Whitelist + /// + /// # Errors + /// + /// Will return a `database::Error` if unable to load the list whitelisted `info_hash`s from the database. + pub async fn load_whitelist_from_database(&self) -> Result<(), databases::error::Error> { + let whitelisted_torrents_from_database = self.database.load_whitelist().await?; + let mut whitelist = self.whitelist.write().await; - db.values().for_each(|torrent_entry| { - let (seeders, completed, leechers) = torrent_entry.get_stats(); - torrents_metrics.seeders += u64::from(seeders); - torrents_metrics.completed += u64::from(completed); - torrents_metrics.leechers += u64::from(leechers); - torrents_metrics.torrents += 1; - }); + whitelist.clear(); - torrents_metrics + for info_hash in whitelisted_torrents_from_database { + let _ = whitelist.insert(info_hash); + } + + Ok(()) } + /// It return the `Tracker` [`statistics::Metrics`]. + /// + /// # Context: Statistics pub async fn get_stats(&self) -> RwLockReadGuard<'_, statistics::Metrics> { self.stats_repository.get_stats().await } + /// It allows to send a statistic events which eventually will be used to update [`statistics::Metrics`]. + /// + /// # Context: Statistics pub async fn send_stats_event(&self, event: statistics::Event) -> Option>> { match &self.stats_event_sender { None => None, Some(stats_event_sender) => stats_event_sender.send_event(event).await, } } - - // Remove inactive peers and (optionally) peerless torrents - pub async fn cleanup_torrents(&self) { - let mut torrents_lock = self.torrents.write().await; - - // If we don't need to remove torrents we will use the faster iter - if self.config.remove_peerless_torrents { - torrents_lock.retain(|_, torrent_entry| { - torrent_entry.remove_inactive_peers(self.config.max_peer_timeout); - - if self.config.persistent_torrent_completed_stat { - torrent_entry.completed > 0 || !torrent_entry.peers.is_empty() - } else { - !torrent_entry.peers.is_empty() - } - }); - } else { - for (_, torrent_entry) in torrents_lock.iter_mut() { - torrent_entry.remove_inactive_peers(self.config.max_peer_timeout); - } - } - } } #[must_use] diff --git a/src/tracker/peer.rs b/src/tracker/peer.rs index 6a298c9df..3626db93d 100644 --- a/src/tracker/peer.rs +++ b/src/tracker/peer.rs @@ -1,3 +1,18 @@ +//! Peer struct used by the core `Tracker`. +//! +//! A sample peer: +//! +//! ```rust,no_run +//! peer::Peer { +//! peer_id: peer::Id(*b"-qB00000000000000000"), +//! peer_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(126, 0, 0, 1)), 8080), +//! updated: DurationSinceUnixEpoch::new(1_669_397_478_934, 0), +//! uploaded: NumberOfBytes(0), +//! downloaded: NumberOfBytes(0), +//! left: NumberOfBytes(0), +//! event: AnnounceEvent::Started, +//! } +//! ``` use std::net::{IpAddr, SocketAddr}; use std::panic::Location; @@ -10,24 +25,49 @@ use crate::shared::bit_torrent::common::{AnnounceEventDef, NumberOfBytesDef}; use crate::shared::clock::utils::ser_unix_time_value; use crate::shared::clock::DurationSinceUnixEpoch; +/// IP version used by the peer to connect to the tracker: IPv4 or IPv6 #[derive(PartialEq, Eq, Debug)] pub enum IPVersion { + /// IPv4, + /// IPv6, } +/// Peer struct used by the core `Tracker`. +/// +/// A sample peer: +/// +/// ```rust,no_run +/// peer::Peer { +/// peer_id: peer::Id(*b"-qB00000000000000000"), +/// peer_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(126, 0, 0, 1)), 8080), +/// updated: DurationSinceUnixEpoch::new(1_669_397_478_934, 0), +/// uploaded: NumberOfBytes(0), +/// downloaded: NumberOfBytes(0), +/// left: NumberOfBytes(0), +/// event: AnnounceEvent::Started, +/// } +/// ``` #[derive(PartialEq, Eq, Debug, Clone, Serialize, Copy)] pub struct Peer { + /// ID used by the downloader peer pub peer_id: Id, + /// The IP and port this peer is listening on pub peer_addr: SocketAddr, + /// The last time the the tracker receive an announce request from this peer (timestamp) #[serde(serialize_with = "ser_unix_time_value")] pub updated: DurationSinceUnixEpoch, + /// The total amount of bytes uploaded by this peer so far #[serde(with = "NumberOfBytesDef")] pub uploaded: NumberOfBytes, + /// The total amount of bytes downloaded by this peer so far #[serde(with = "NumberOfBytesDef")] pub downloaded: NumberOfBytes, + /// The number of bytes this peer still has to download #[serde(with = "NumberOfBytesDef")] - pub left: NumberOfBytes, // The number of bytes this peer still has to download + pub left: NumberOfBytes, + /// This is an optional key which maps to started, completed, or stopped (or empty, which is the same as not being present). #[serde(with = "AnnounceEventDef")] pub event: AnnounceEvent, } @@ -56,11 +96,24 @@ impl Peer { } } +/// Peer ID. A 20-byte array. +/// +/// A string of length 20 which this downloader uses as its id. +/// Each downloader generates its own id at random at the start of a new download. +/// +/// A sample peer ID: +/// +/// ```rust,no_run +/// let peer_id = peer::Id(*b"-qB00000000000000000"); +/// ``` #[derive(PartialEq, Eq, Hash, Clone, Debug, PartialOrd, Ord, Copy)] pub struct Id(pub [u8; 20]); const PEER_ID_BYTES_LEN: usize = 20; +/// Error returned when trying to convert an invalid peer id from another type. +/// +/// Usually because the source format does not contain 20 bytes. #[derive(Error, Debug)] pub enum IdConversionError { #[error("not enough bytes for peer id: {message} {location}")] diff --git a/src/tracker/services/mod.rs b/src/tracker/services/mod.rs index 8667f79a9..deb07a439 100644 --- a/src/tracker/services/mod.rs +++ b/src/tracker/services/mod.rs @@ -1,3 +1,9 @@ +//! Tracker domain services. Core and statistics services. +//! +//! There are two types of service: +//! +//! - [Core tracker services](crate::tracker::services::torrent): related to the tracker main functionalities like getting info about torrents. +//! - [Services for statistics](crate::tracker::services::statistics): related to tracker metrics. Aggregate data about the tracker server. pub mod statistics; pub mod torrent; @@ -7,6 +13,8 @@ use torrust_tracker_configuration::Configuration; use crate::tracker::Tracker; +/// It returns a new tracker building its dependencies. +/// /// # Panics /// /// Will panic if tracker cannot be instantiated. diff --git a/src/tracker/services/statistics/mod.rs b/src/tracker/services/statistics/mod.rs index cae4d1d69..143761420 100644 --- a/src/tracker/services/statistics/mod.rs +++ b/src/tracker/services/statistics/mod.rs @@ -1,3 +1,41 @@ +//! Statistics services. +//! +//! It includes: +//! +//! - A [`factory`](crate::tracker::services::statistics::setup::factory) function to build the structs needed to collect the tracker metrics. +//! - A [`get_metrics`](crate::tracker::services::statistics::get_metrics) service to get the [`tracker metrics`](crate::tracker::statistics::Metrics). +//! +//! Tracker metrics are collected using a Publisher-Subscribe pattern. +//! +//! The factory function builds two structs: +//! +//! - An statistics [`EventSender`](crate::tracker::statistics::EventSender) +//! - An statistics [`Repo`](crate::tracker::statistics::Repo) +//! +//! ```rust,no_run +//! let (stats_event_sender, stats_repository) = factory(tracker_usage_statistics); +//! ``` +//! +//! The statistics repository is responsible for storing the metrics in memory. +//! The statistics event sender allows sending events related to metrics. +//! There is an event listener that is receiving all the events and processing them with an event handler. +//! Then, the event handler updates the metrics depending on the received event. +//! +//! For example, if you send the event [`Event::Udp4Connect`](crate::tracker::statistics::Event::Udp4Connect): +//! +//! ```rust,no_run +//! let result = event_sender.send_event(Event::Udp4Connect).await; +//! ``` +//! +//! Eventually the counter for UDP connections from IPv4 peers will be increased. +//! +//! ```rust,no_run +//! pub struct Metrics { +//! // ... +//! pub udp4_connections_handled: u64, // This will be incremented +//! // ... +//! } +//! ``` pub mod setup; use std::sync::Arc; @@ -5,12 +43,21 @@ use std::sync::Arc; use crate::tracker::statistics::Metrics; use crate::tracker::{TorrentsMetrics, Tracker}; +/// All the metrics collected by the tracker. #[derive(Debug, PartialEq)] pub struct TrackerMetrics { + /// Domain level metrics. + /// + /// General metrics for all torrents (number of seeders, leechers, etcetera) pub torrents_metrics: TorrentsMetrics, + + /// Application level metrics. Usage statistics/metrics. + /// + /// Metrics about how the tracker is been used (number of udp announce requests, number of http scrape requests, etcetera) pub protocol_metrics: Metrics, } +/// It returns all the [`TrackerMetrics`](crate::tracker::services::statistics::TrackerMetrics) pub async fn get_metrics(tracker: Arc) -> TrackerMetrics { let torrents_metrics = tracker.get_torrents_metrics().await; let stats = tracker.get_stats().await; diff --git a/src/tracker/services/statistics/setup.rs b/src/tracker/services/statistics/setup.rs index b7cb831cb..b8d325ab4 100644 --- a/src/tracker/services/statistics/setup.rs +++ b/src/tracker/services/statistics/setup.rs @@ -1,5 +1,17 @@ +//! Setup for the tracker statistics. +//! +//! The [`factory`](crate::tracker::services::statistics::setup::factory) function builds the structs needed for handling the tracker metrics. use crate::tracker::statistics; +/// It builds the structs needed for handling the tracker metrics. +/// +/// It returns: +/// +/// - An statistics [`EventSender`](crate::tracker::statistics::EventSender) that allows you to send events related to statistics. +/// - An statistics [`Repo`](crate::tracker::statistics::Repo) which is an in-memory repository for the tracker metrics. +/// +/// When the input argument `tracker_usage_statistics`is false the setup does not run the event listeners, consequently the statistics +/// events are sent are received but not dispatched to the handler. #[must_use] pub fn factory(tracker_usage_statistics: bool) -> (Option>, statistics::Repo) { let mut stats_event_sender = None; diff --git a/src/tracker/services/torrent.rs b/src/tracker/services/torrent.rs index 30d24eb00..3610d930c 100644 --- a/src/tracker/services/torrent.rs +++ b/src/tracker/services/torrent.rs @@ -1,3 +1,9 @@ +//! Core tracker domain services. +//! +//! There are two services: +//! +//! - [`get_torrent_info`](crate::tracker::services::torrent::get_torrent_info): it returns all the data about one torrent. +//! - [`get_torrents`](crate::tracker::services::torrent::get_torrents): it returns data about some torrent in bulk excluding the peer list. use std::sync::Arc; use serde::Deserialize; @@ -6,26 +12,42 @@ use crate::shared::bit_torrent::info_hash::InfoHash; use crate::tracker::peer::Peer; use crate::tracker::Tracker; +/// It contains all the information the tracker has about a torrent #[derive(Debug, PartialEq)] pub struct Info { + /// The infohash of the torrent this data is related to pub info_hash: InfoHash, + /// The total number of seeders for this torrent. Peer that actively serving a full copy of the torrent data pub seeders: u64, + /// The total number of peers that have ever complete downloading this torrent pub completed: u64, + /// The total number of leechers for this torrent. Peers that actively downloading this torrent pub leechers: u64, + /// The swarm: the list of peers that are actively trying to download or serving this torrent pub peers: Option>, } +/// It contains only part of the information the tracker has about a torrent +/// +/// It contains the same data as [Info](crate::tracker::services::torrent::Info) but without the list of peers in the swarm. #[derive(Debug, PartialEq, Clone)] pub struct BasicInfo { + /// The infohash of the torrent this data is related to pub info_hash: InfoHash, + /// The total number of seeders for this torrent. Peer that actively serving a full copy of the torrent data pub seeders: u64, + /// The total number of peers that have ever complete downloading this torrent pub completed: u64, + /// The total number of leechers for this torrent. Peers that actively downloading this torrent pub leechers: u64, } +/// A struct to keep information about the page when results are being paginated #[derive(Deserialize)] pub struct Pagination { + /// The page number, starting at 0 pub offset: u32, + /// Page size. The number of results per page pub limit: u32, } @@ -69,6 +91,7 @@ impl Default for Pagination { } } +/// It returns all the information the tracker has about one torrent in a [Info](crate::tracker::services::torrent::Info) struct. pub async fn get_torrent_info(tracker: Arc, info_hash: &InfoHash) -> Option { let db = tracker.get_torrents().await; @@ -93,6 +116,7 @@ pub async fn get_torrent_info(tracker: Arc, info_hash: &InfoHash) -> Op }) } +/// It returns all the information the tracker has about multiple torrents in a [`BasicInfo`](crate::tracker::services::torrent::BasicInfo) struct, excluding the peer list. pub async fn get_torrents(tracker: Arc, pagination: &Pagination) -> Vec { let db = tracker.get_torrents().await; diff --git a/src/tracker/statistics.rs b/src/tracker/statistics.rs index f9079962c..03f4fc081 100644 --- a/src/tracker/statistics.rs +++ b/src/tracker/statistics.rs @@ -1,3 +1,22 @@ +//! Structs to collect and keep tracker metrics. +//! +//! The tracker collects metrics such as: +//! +//! - Number of connections handled +//! - Number of `announce` requests handled +//! - Number of `scrape` request handled +//! +//! These metrics are collected for each connection type: UDP and HTTP and +//! also for each IP version used by the peers: IPv4 and IPv6. +//! +//! > Notice: that UDP tracker have an specific `connection` request. For the HTTP metrics the counter counts one connection for each `announce` or `scrape` request. +//! +//! The data is collected by using an `event-sender -> event listener` model. +//! +//! The tracker uses an [`statistics::EventSender`](crate::tracker::statistics::EventSender) instance to send an event. +//! The [`statistics::Keeper`](crate::tracker::statistics::Keeper) listens to new events and uses the [`statistics::Repo`](crate::tracker::statistics::Repo) to upgrade and store metrics. +//! +//! See the [`statistics::Event`](crate::tracker::statistics::Event) enum to check which events are available. use std::sync::Arc; use async_trait::async_trait; @@ -9,6 +28,14 @@ use tokio::sync::{mpsc, RwLock, RwLockReadGuard}; const CHANNEL_BUFFER_SIZE: usize = 65_535; +/// An statistics event. It is used to collect tracker metrics. +/// +/// - `Tcp` prefix means the event was triggered by the HTTP tracker +/// - `Udp` prefix means the event was triggered by the UDP tracker +/// - `4` or `6` prefixes means the IP version used by the peer +/// - Finally the event suffix is the type of request: `announce`, `scrape` or `connection` +/// +/// > NOTE: HTTP trackers do not use `connection` requests. #[derive(Debug, PartialEq, Eq)] pub enum Event { // code-review: consider one single event for request type with data: Event::Announce { scheme: HTTPorUDP, ip_version: V4orV6 } @@ -25,6 +52,14 @@ pub enum Event { Udp6Scrape, } +/// Metrics collected by the tracker. +/// +/// - Number of connections handled +/// - Number of `announce` requests handled +/// - Number of `scrape` request handled +/// +/// These metrics are collected for each connection type: UDP and HTTP +/// and also for each IP version used by the peers: IPv4 and IPv6. #[derive(Debug, PartialEq, Default)] pub struct Metrics { pub tcp4_connections_handled: u64, @@ -41,6 +76,10 @@ pub struct Metrics { pub udp6_scrapes_handled: u64, } +/// The service responsible for keeping tracker metrics (listening to statistics events and handle them). +/// +/// It actively listen to new statistics events. When it receives a new event +/// it accordingly increases the counters. pub struct Keeper { pub repository: Repo, } @@ -131,12 +170,17 @@ async fn event_handler(event: Event, stats_repository: &Repo) { debug!("stats: {:?}", stats_repository.get_stats().await); } +/// A trait to allow sending statistics events #[async_trait] #[cfg_attr(test, automock)] pub trait EventSender: Sync + Send { async fn send_event(&self, event: Event) -> Option>>; } +/// An [`statistics::EventSender`](crate::tracker::statistics::EventSender) implementation. +/// +/// It uses a channel sender to send the statistic events. The channel is created by a +/// [`statistics::Keeper`](crate::tracker::statistics::Keeper) pub struct Sender { sender: mpsc::Sender, } @@ -148,6 +192,7 @@ impl EventSender for Sender { } } +/// A repository for the tracker metrics. #[derive(Clone)] pub struct Repo { pub stats: Arc>, diff --git a/src/tracker/torrent.rs b/src/tracker/torrent.rs index 882e52ff1..8eb557f1e 100644 --- a/src/tracker/torrent.rs +++ b/src/tracker/torrent.rs @@ -1,3 +1,33 @@ +//! Structs to store the swarm data. +//! +//! There are to main data structures: +//! +//! - A torrent [`Entry`](crate::tracker::torrent::Entry): it contains all the information stored by the tracker for one torrent. +//! - The [`SwarmMetadata`](crate::tracker::torrent::SwarmMetadata): it contains aggregate information that can me derived from the torrent entries. +//! +//! A "swarm" is a network of peers that are trying to download the same torrent. +//! +//! The torrent entry contains the "swarm" data, which is basically the list of peers in the swarm. +//! That's the most valuable information the peer want to get from the tracker, because it allows them to +//! start downloading torrent from those peers. +//! +//! > **NOTICE**: that both swarm data (torrent entries) and swarm metadata (aggregate counters) are related to only one torrent. +//! +//! The "swarm metadata" contains aggregate data derived from the torrent entries. There two types of data: +//! +//! - For **active peers**: metrics related to the current active peers in the swarm. +//! - **Historical data**: since the tracker started running. +//! +//! The tracker collects metrics for: +//! +//! - The number of peers that have completed downloading the torrent since the tracker started collecting metrics. +//! - The number of peers that have completed downloading the torrent and are still active, that means they are actively participating in the network, +//! by announcing themselves periodically to the tracker. Since they have completed downloading they have a full copy of the torrent data. Peers with a +//! full copy of the data are called "seeders". +//! - The number of peers that have NOT completed downloading the torrent and are still active, that means they are actively participating in the network. +//! Peer that don not have a full copy of the torrent data are called "leechers". +//! +//! > **NOTICE**: that both [`SwarmMetadata`](crate::tracker::torrent::SwarmMetadata) and [`SwarmStats`](crate::tracker::torrent::SwarmStats) contain the same information. [`SwarmMetadata`](crate::tracker::torrent::SwarmMetadata) is using the names used on [BEP 48: Tracker Protocol Extension: Scrape](https://www.bittorrent.org/beps/bep_0048.html). use std::time::Duration; use aquatic_udp_protocol::AnnounceEvent; @@ -7,21 +37,33 @@ use super::peer::{self, Peer}; use crate::shared::bit_torrent::common::MAX_SCRAPE_TORRENTS; use crate::shared::clock::{Current, TimeNow}; +/// A data structure containing all the information about a torrent in the tracker. +/// +/// This is the tracker entry for a given torrent and contains the swarm data, +/// that's the list of all the peers trying to download the same torrent. +/// The tracker keeps one entry like this for every torrent. #[derive(Serialize, Deserialize, Clone, Debug)] pub struct Entry { + /// The swarm: a network of peers that are all trying to download the torrent associated to this entry #[serde(skip)] pub peers: std::collections::BTreeMap, + /// The number of peers that have ever completed downloading the torrent associated to this entry pub completed: u32, } /// Swarm statistics for one torrent. /// Swarm metadata dictionary in the scrape response. -/// BEP 48: +/// +/// See [BEP 48: Tracker Protocol Extension: Scrape](https://www.bittorrent.org/beps/bep_0048.html) #[derive(Debug, PartialEq, Default)] pub struct SwarmMetadata { - pub complete: u32, // The number of active peers that have completed downloading (seeders) - pub downloaded: u32, // The number of peers that have ever completed downloading - pub incomplete: u32, // The number of active peers that have not completed downloading (leechers) + /// The number of peers that have ever completed downloading + pub downloaded: u32, + + /// The number of active peers that have completed downloading (seeders) + pub complete: u32, + /// The number of active peers that have not completed downloading (leechers) + pub incomplete: u32, } impl SwarmMetadata { @@ -32,12 +74,17 @@ impl SwarmMetadata { } /// Swarm statistics for one torrent. -/// Alternative struct for swarm metadata in scrape response. +/// +/// See [BEP 48: Tracker Protocol Extension: Scrape](https://www.bittorrent.org/beps/bep_0048.html) #[derive(Debug, PartialEq, Default)] pub struct SwarmStats { - pub completed: u32, // The number of peers that have ever completed downloading - pub seeders: u32, // The number of active peers that have completed downloading (seeders) - pub leechers: u32, // The number of active peers that have not completed downloading (leechers) + /// The number of peers that have ever completed downloading + pub completed: u32, + + /// The number of active peers that have completed downloading (seeders) + pub seeders: u32, + /// The number of active peers that have not completed downloading (leechers) + pub leechers: u32, } impl Entry { @@ -49,7 +96,10 @@ impl Entry { } } - // Update peer and return completed (times torrent has been downloaded) + /// It updates a peer and returns true if the number of complete downloads have increased. + /// + /// The number of peers that have complete downloading is synchronously updated when peers are updated. + /// That's the total torrent downloads counter. pub fn update_peer(&mut self, peer: &peer::Peer) -> bool { let mut did_torrent_stats_change: bool = false; @@ -73,14 +123,15 @@ impl Entry { did_torrent_stats_change } - /// Get all peers, limiting the result to the maximum number of scrape torrents. + /// Get all swarm peers, limiting the result to the maximum number of scrape torrents. #[must_use] pub fn get_all_peers(&self) -> Vec<&peer::Peer> { self.peers.values().take(MAX_SCRAPE_TORRENTS as usize).collect() } - /// Returns the list of peers for a given client. - /// It filters out the input peer. + /// It returns the list of peers for a given peer client. + /// + /// It filters out the input peer, typically because we want to return this list of peers to that client peer. #[must_use] pub fn get_peers_for_peer(&self, client: &Peer) -> Vec<&peer::Peer> { self.peers @@ -92,6 +143,7 @@ impl Entry { .collect() } + /// It returns the swarm metadata (statistics) as a tuple `(seeders, completed, leechers)` #[allow(clippy::cast_possible_truncation)] #[must_use] pub fn get_stats(&self) -> (u32, u32, u32) { @@ -100,6 +152,7 @@ impl Entry { (seeders, self.completed, leechers) } + /// It returns the swarm metadata (statistics) as an struct #[must_use] pub fn get_swarm_metadata(&self) -> SwarmMetadata { // code-review: consider using always this function instead of `get_stats`. @@ -111,6 +164,7 @@ impl Entry { } } + /// It removes peer from the swarm that have not been updated for more than `max_peer_timeout` seconds pub fn remove_inactive_peers(&mut self, max_peer_timeout: u32) { let current_cutoff = Current::sub(&Duration::from_secs(u64::from(max_peer_timeout))).unwrap_or_default(); self.peers.retain(|_, peer| peer.updated > current_cutoff);