Merge #269: feat: [#261] store original info-hashes

110e159 test: [#261]: do not allow uploading two torrents with the same canonical infohash (Jose Celano)
3b7a762 feat: [#261] store original infohashes (Jose Celano)

Pull request description:

  When you upload a torrent, the infohash might change if the `info` dictionary contains non-standard fields because we remove them. That leads to a different infohash. We store the
   original infohash in a new table so that we can know if the torrent was previously uploaded.

  If we do not store the original infohash we could reject uploads producing the same canonical infohash. Still, there is no way for the user to ask if a torrent exists with a given original infohash. They only would be able to interact with the API with the canonical infohash.

  Sometimes it's useful to use the original infohash, for instance, if you are importing torrents from an external source and you want to check if the original torrent (with the original infohash) was already uploaded.

ACKs for top commit:
  josecelano:
    ACK 110e159

Tree-SHA512: 867ef055d51b43f602006df1d7ea9054918505b1603c572eb717bb6a98740ecea3c83f3dcd7c505e70b98a6fb70b5f22b83c8a7e0a4424e130ab1f17aa450dcc

.github/workflows/coverage.yaml

@@ -2,24 +2,55 @@ name: Coverage
 
 on:
   push:
-  pull_request:
+    branches:
+      - develop
+  pull_request_target:
+    branches:
+      - develop
 
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
+  secrets:
+    name: Secrets
+    environment: coverage
+    runs-on: ubuntu-latest
+
+    outputs:
+      continue: ${{ steps.check.outputs.continue }}
+
+    steps:
+      - id: check
+        name: Check
+        env:
+          CODECOV_TOKEN: "${{ secrets.CODECOV_TOKEN }}"
+        if: "${{ env.CODECOV_TOKEN != '' }}"
+        run: echo "continue=true" >> $GITHUB_OUTPUT
+
   report:
     name: Report
+    environment: coverage
+    needs: secrets
+    if: needs.secrets.outputs.continue == 'true'
     runs-on: ubuntu-latest
     env:
       CARGO_INCREMENTAL: "0"
       RUSTFLAGS: "-Z profile -C codegen-units=1 -C inline-threshold=0 -C link-dead-code -C overflow-checks=off -C panic=abort -Z panic_abort_tests"
       RUSTDOCFLAGS: "-Z profile -C codegen-units=1 -C inline-threshold=0 -C link-dead-code -C overflow-checks=off -C panic=abort -Z panic_abort_tests"
 
     steps:
-      - id: checkout
-        name: Checkout Repository
-        uses: actions/checkout@v3
+      - id: checkout_push
+        if: github.event_name  == 'push'
+        name: Checkout Repository (Push)
+        uses: actions/checkout@v4
+
+      - id: checkout_pull_request_target
+        if: github.event_name  == 'pull_request_target'
+        name: Checkout Repository (Pull Request Target)
+        uses: actions/checkout@v4
+        with:
+          ref: "refs/pull/${{ github.event.pull_request.number }}/head"
 
       - id: setup
         name: Setup Toolchain
@@ -58,6 +89,7 @@ jobs:
         name: Upload Coverage Report
         uses: codecov/codecov-action@v3
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           files: ${{ steps.coverage.outputs.report }}
           verbose: true
-          fail_ci_if_error: true
+          fail_ci_if_error: true

.github/workflows/testing.yaml

@@ -65,10 +65,14 @@ jobs:
         name: Run Lint Checks
         run: cargo clippy --tests --benches --examples --workspace --all-targets --all-features -- -D clippy::correctness -D clippy::suspicious -D clippy::complexity -D clippy::perf -D clippy::style -D clippy::pedantic
 
-      - id: doc
-        name: Run Documentation Checks
+      - id: testdoc
+        name: Run Documentation Tests
         run: cargo test --doc
 
+      - id: builddoc
+        name: Build Documentation
+        run: cargo doc --no-deps --bins --examples --workspace --all-features
+
   unit:
     name: Units
     runs-on: ubuntu-latest

migrations/mysql/20230905091837_torrust_multiple_original_infohashes.sql

@@ -0,0 +1,46 @@
+-- Step 1: Create a new table with all infohashes
+CREATE TABLE torrust_torrent_info_hashes (
+    info_hash CHAR(40) NOT NULL,
+    canonical_info_hash CHAR(40) NOT NULL,
+    original_is_known BOOLEAN NOT NULL,
+    PRIMARY KEY(info_hash),
+    FOREIGN KEY(canonical_info_hash) REFERENCES torrust_torrents(info_hash) ON DELETE CASCADE
+);
+
+-- Step 2: Create one record for each torrent with only the canonical infohash.
+--         The original infohash is NULL so we do not know if it was the same.
+--         This happens if the uploaded torrent was uploaded before introducing
+--         the feature to store the original infohash
+INSERT INTO torrust_torrent_info_hashes (info_hash, canonical_info_hash, original_is_known)
+SELECT info_hash, info_hash, FALSE
+    FROM torrust_torrents
+    WHERE original_info_hash IS NULL;
+
+-- Step 3: Create one record for each torrent with the same original and 
+--         canonical infohashes.
+INSERT INTO torrust_torrent_info_hashes (info_hash, canonical_info_hash, original_is_known)
+SELECT info_hash, info_hash, TRUE
+    FROM torrust_torrents
+    WHERE original_info_hash IS NOT NULL
+        AND info_hash = original_info_hash;
+
+-- Step 4: Create two records for each torrent with a different original and 
+--         canonical infohashes. One record with the same original and canonical
+--         infohashes and one record with the original infohash and the canonical
+--         one.
+-- Insert the canonical infohash
+INSERT INTO torrust_torrent_info_hashes (info_hash, canonical_info_hash, original_is_known)
+SELECT info_hash, info_hash, TRUE
+    FROM torrust_torrents
+    WHERE original_info_hash IS NOT NULL
+        AND info_hash != original_info_hash;
+-- Insert the original infohash pointing to the canonical
+INSERT INTO torrust_torrent_info_hashes (info_hash, canonical_info_hash, original_is_known)
+SELECT original_info_hash, info_hash, TRUE
+    FROM torrust_torrents
+    WHERE original_info_hash IS NOT NULL
+        AND info_hash != original_info_hash;
+
+-- Step 5: Delete the `torrust_torrents::original_info_hash` column
+ALTER TABLE torrust_torrents DROP COLUMN original_info_hash;
+

migrations/sqlite3/20230905091837_torrust_multiple_original_infohashes.sql

@@ -0,0 +1,48 @@
+-- Step 1: Create a new table with all infohashes
+CREATE TABLE IF NOT EXISTS torrust_torrent_info_hashes (
+	info_hash TEXT NOT NULL,
+	canonical_info_hash	TEXT NOT NULL,
+    original_is_known BOOLEAN NOT NULL,
+	PRIMARY KEY(info_hash),
+	FOREIGN KEY(canonical_info_hash) REFERENCES torrust_torrents (info_hash) ON DELETE CASCADE
+);
+
+-- Step 2: Create one record for each torrent with only the canonical infohash.
+--         The original infohash is NULL so we do not know if it was the same.
+--         This happens if the uploaded torrent was uploaded before introducing
+--         the feature to store the original infohash
+INSERT INTO torrust_torrent_info_hashes (info_hash, canonical_info_hash, original_is_known)
+SELECT info_hash, info_hash, FALSE
+    FROM torrust_torrents
+    WHERE original_info_hash is NULL;
+
+-- Step 3: Create one record for each torrent with the same original and 
+--         canonical infohashes.
+INSERT INTO torrust_torrent_info_hashes (info_hash, canonical_info_hash, original_is_known)
+SELECT info_hash, info_hash, TRUE
+    FROM torrust_torrents
+    WHERE original_info_hash is NOT NULL
+        AND info_hash = original_info_hash;
+
+-- Step 4: Create two records for each torrent with a different original and 
+--         canonical infohashes. One record with the same original and canonical
+--         infohashes and one record with the original infohash and the canonical
+--         one.
+-- Insert the canonical infohash
+INSERT INTO torrust_torrent_info_hashes (info_hash, canonical_info_hash, original_is_known)
+SELECT info_hash, info_hash, TRUE
+    FROM torrust_torrents
+    WHERE original_info_hash is NOT NULL
+        AND info_hash != original_info_hash;
+-- Insert the original infohash pointing to the canonical
+INSERT INTO torrust_torrent_info_hashes (info_hash, canonical_info_hash, original_is_known)
+SELECT original_info_hash, info_hash, TRUE
+    FROM torrust_torrents
+    WHERE original_info_hash is NOT NULL
+        AND info_hash != original_info_hash;
+
+-- Step 5: Delete the `torrust_torrents::original_info_hash` column
+--         SQLite 2021-03-12 (3.35.0) supports DROP COLUMN
+--         https://www.sqlite.org/lang_altertable.html#alter_table_drop_column
+ALTER TABLE torrust_torrents DROP COLUMN original_info_hash;
+

src/app.rs

@@ -12,8 +12,8 @@ use crate::services::authentication::{DbUserAuthenticationRepository, JsonWebTok
 use crate::services::category::{self, DbCategoryRepository};
 use crate::services::tag::{self, DbTagRepository};
 use crate::services::torrent::{
-    DbTorrentAnnounceUrlRepository, DbTorrentFileRepository, DbTorrentInfoRepository, DbTorrentListingGenerator,
-    DbTorrentRepository, DbTorrentTagRepository,
+    DbTorrentAnnounceUrlRepository, DbTorrentFileRepository, DbTorrentInfoHashRepository, DbTorrentInfoRepository,
+    DbTorrentListingGenerator, DbTorrentRepository, DbTorrentTagRepository,
 };
 use crate::services::user::{self, DbBannedUserList, DbUserProfileRepository, DbUserRepository};
 use crate::services::{proxy, settings, torrent};
@@ -68,6 +68,7 @@ pub async fn run(configuration: Configuration, api_version: &Version) -> Running
     let user_authentication_repository = Arc::new(DbUserAuthenticationRepository::new(database.clone()));
     let user_profile_repository = Arc::new(DbUserProfileRepository::new(database.clone()));
     let torrent_repository = Arc::new(DbTorrentRepository::new(database.clone()));
+    let torrent_info_hash_repository = Arc::new(DbTorrentInfoHashRepository::new(database.clone()));
     let torrent_info_repository = Arc::new(DbTorrentInfoRepository::new(database.clone()));
     let torrent_file_repository = Arc::new(DbTorrentFileRepository::new(database.clone()));
     let torrent_announce_url_repository = Arc::new(DbTorrentAnnounceUrlRepository::new(database.clone()));
@@ -92,6 +93,7 @@ pub async fn run(configuration: Configuration, api_version: &Version) -> Running
         user_repository.clone(),
         category_repository.clone(),
         torrent_repository.clone(),
+        torrent_info_hash_repository.clone(),
         torrent_info_repository.clone(),
         torrent_file_repository.clone(),
         torrent_announce_url_repository.clone(),
@@ -135,6 +137,7 @@ pub async fn run(configuration: Configuration, api_version: &Version) -> Running
         user_authentication_repository,
         user_profile_repository,
         torrent_repository,
+        torrent_info_hash_repository,
         torrent_info_repository,
         torrent_file_repository,
         torrent_announce_url_repository,

src/common.rs

@@ -7,8 +7,8 @@ use crate::services::authentication::{DbUserAuthenticationRepository, JsonWebTok
 use crate::services::category::{self, DbCategoryRepository};
 use crate::services::tag::{self, DbTagRepository};
 use crate::services::torrent::{
-    DbTorrentAnnounceUrlRepository, DbTorrentFileRepository, DbTorrentInfoRepository, DbTorrentListingGenerator,
-    DbTorrentRepository, DbTorrentTagRepository,
+    DbTorrentAnnounceUrlRepository, DbTorrentFileRepository, DbTorrentInfoHashRepository, DbTorrentInfoRepository,
+    DbTorrentListingGenerator, DbTorrentRepository, DbTorrentTagRepository,
 };
 use crate::services::user::{self, DbBannedUserList, DbUserProfileRepository, DbUserRepository};
 use crate::services::{proxy, settings, torrent};
@@ -34,6 +34,7 @@ pub struct AppData {
     pub user_authentication_repository: Arc<DbUserAuthenticationRepository>,
     pub user_profile_repository: Arc<DbUserProfileRepository>,
     pub torrent_repository: Arc<DbTorrentRepository>,
+    pub torrent_info_hash_repository: Arc<DbTorrentInfoHashRepository>,
     pub torrent_info_repository: Arc<DbTorrentInfoRepository>,
     pub torrent_file_repository: Arc<DbTorrentFileRepository>,
     pub torrent_announce_url_repository: Arc<DbTorrentAnnounceUrlRepository>,
@@ -69,6 +70,7 @@ impl AppData {
         user_authentication_repository: Arc<DbUserAuthenticationRepository>,
         user_profile_repository: Arc<DbUserProfileRepository>,
         torrent_repository: Arc<DbTorrentRepository>,
+        torrent_info_hash_repository: Arc<DbTorrentInfoHashRepository>,
         torrent_info_repository: Arc<DbTorrentInfoRepository>,
         torrent_file_repository: Arc<DbTorrentFileRepository>,
         torrent_announce_url_repository: Arc<DbTorrentAnnounceUrlRepository>,
@@ -101,6 +103,7 @@ impl AppData {
             user_authentication_repository,
             user_profile_repository,
             torrent_repository,
+            torrent_info_hash_repository,
             torrent_info_repository,
             torrent_file_repository,
             torrent_announce_url_repository,

src/databases/database.rs

@@ -12,6 +12,7 @@ use crate::models::torrent_file::{DbTorrentInfo, Torrent, TorrentFile};
 use crate::models::torrent_tag::{TagId, TorrentTag};
 use crate::models::tracker_key::TrackerKey;
 use crate::models::user::{User, UserAuthentication, UserCompact, UserId, UserProfile};
+use crate::services::torrent::OriginalInfoHashes;
 
 /// Database tables to be truncated when upgrading from v1.0.0 to v2.0.0.
 /// They must be in the correct order to avoid foreign key errors.
@@ -87,6 +88,7 @@ pub enum Error {
     TorrentNotFound,
     TorrentAlreadyExists, // when uploading an already uploaded info_hash
     TorrentTitleAlreadyExists,
+    TorrentInfoHashNotFound,
 }
 
 /// Get the Driver of the Database from the Connection String
@@ -229,6 +231,20 @@ pub trait Database: Sync + Send {
         ))
     }
 
+    /// Returns the list of all infohashes producing the same canonical infohash.
+    ///
+    /// When you upload a torrent the infohash migth change because the Index
+    /// remove the non-standard fields in the `info` dictionary. That makes the
+    /// infohash change. The canonical infohash is the resulting infohash.
+    /// This function returns the original infohashes of a canonical infohash.
+    ///
+    /// If the original infohash was unknown, it returns the canonical infohash.
+    ///
+    /// The relationship is 1 canonical infohash -> N original infohashes.
+    async fn get_torrent_canonical_info_hash_group(&self, canonical: &InfoHash) -> Result<OriginalInfoHashes, Error>;
+
+    async fn insert_torrent_info_hash(&self, original: &InfoHash, canonical: &InfoHash) -> Result<(), Error>;
+
     /// Get torrent's info as `DbTorrentInfo` from `torrent_id`.
     async fn get_torrent_info_from_id(&self, torrent_id: i64) -> Result<DbTorrentInfo, Error>;