Merge #252: Run E2E tests also using MySQL

0b29678 refactor(ci): reorganize E2E testing scripts (Jose Celano)
22b8f8a fix: [#223] HTTP error status code trying to insert duplicate category in MySQL (Jose Celano)
c6346a5 test: [#223] run E2E with MySQL too (Jose Celano)

Pull request description:

  E2E tests are only executed with SQLite.

Top commit has no ACKs.

Tree-SHA512: 07e0197e9fd8e5341511ed3097ef364e6148bf282bd09382bdbda878eeaaa6d4f331d683cb15a7eadfa4cb7fe51df45e2424ddb3514c990d98875717c755588f

.github/workflows/develop.yml

@@ -30,4 +30,5 @@ jobs:
       - name: Test Coverage
         run: cargo llvm-cov nextest
       - name: E2E Tests
-        run: ./docker/bin/run-e2e-tests.sh
+        run: ./docker/bin/e2e/run-e2e-tests.sh
+

compose.yaml

@@ -87,7 +87,7 @@ services:
     environment:
       - MYSQL_ROOT_HOST=%
       - MYSQL_ROOT_PASSWORD=root_secret_password
-      - MYSQL_DATABASE=torrust_index_backend
+      - MYSQL_DATABASE=${TORRUST_IDX_BACK_MYSQL_DATABASE:-torrust_index_backend_e2e_testing}
       - MYSQL_USER=db_user
       - MYSQL_PASSWORD=db_user_secret_password
     networks:

config-idx-back.mysql.local.toml

@@ -0,0 +1,46 @@
+log_level = "info"
+
+[website]
+name = "Torrust"
+
+[tracker]
+url = "udp://tracker:6969"
+mode = "Public"
+api_url = "http://tracker:1212"
+token = "MyAccessToken"
+token_valid_seconds = 7257600
+
+[net]
+port = 3001
+
+[auth]
+email_on_signup = "Optional"
+min_password_length = 6
+max_password_length = 64
+secret_key = "MaxVerstappenWC2021"
+
+[database]
+connect_url = "mysql://root:root_secret_password@mysql:3306/torrust_index_backend_e2e_testing"
+
+[mail]
+email_verification_enabled = false
+from = "[email protected]"
+reply_to = "[email protected]"
+username = ""
+password = ""
+server = "mailcatcher"
+port = 1025
+
+[image_cache]
+max_request_timeout_ms = 1000
+capacity = 128000000
+entry_size_limit = 4000000
+user_quota_period_seconds = 3600
+user_quota_bytes = 64000000
+
+[api]
+default_torrent_page_size = 10
+max_torrent_page_size = 30
+
+[tracker_statistics_importer]
+torrent_info_update_interval = 3600

config-idx-back.local.toml → config-idx-back.sqlite.local.toml

@@ -20,8 +20,7 @@ max_password_length = 64
 secret_key = "MaxVerstappenWC2021"
 
 [database]
-connect_url = "sqlite://storage/database/torrust_index_backend_e2e_testing.db?mode=rwc" # SQLite
-#connect_url = "mysql://root:root_secret_password@mysql:3306/torrust_index_backend" # MySQL
+connect_url = "sqlite://storage/database/torrust_index_backend_e2e_testing.db?mode=rwc"
 
 [mail]
 email_verification_enabled = false

docker/bin/e2e/mysql/e2e-env-reset.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Delete the databases and recreate them.
+
+docker compose down
+
+# Index Backend
+
+# Database credentials
+MYSQL_USER="root"
+MYSQL_PASSWORD="root_secret_password"
+MYSQL_HOST="localhost"
+MYSQL_DATABASE="torrust_index_backend_e2e_testing"
+
+# Create the MySQL database for the index backend. Assumes MySQL client is installed.
+echo "Creating MySQL database $MYSQL_DATABASE for E2E testing ..."
+mysql -h $MYSQL_HOST -u $MYSQL_USER -p$MYSQL_PASSWORD -e "DROP DATABASE IF EXISTS $MYSQL_DATABASE; CREATE DATABASE $MYSQL_DATABASE;"
+
+# Tracker
+
+# Delete tracker database
+rm -f ./storage/database/torrust_tracker_e2e_testing.db
+
+# Generate storage directory if it does not exist
+mkdir -p "./storage/database"
+
+# Generate the sqlite database for the tracker if it does not exist
+if ! [ -f "./storage/database/torrust_tracker_e2e_testing.db" ]; then
+    touch ./storage/database/torrust_tracker_e2e_testing.db
+    echo ";" | sqlite3 ./storage/database/torrust_tracker_e2e_testing.db
+fi
+
+./docker/bin/e2e/mysql/e2e-env-up.sh

docker/bin/e2e/mysql/e2e-env-restart.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+docker compose down
+./docker/bin/e2e/mysql/e2e-env-up.sh

docker/bin/e2e/mysql/e2e-env-up.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+TORRUST_IDX_BACK_USER_UID=${TORRUST_IDX_BACK_USER_UID:-1000} \
+    docker compose build
+
+TORRUST_IDX_BACK_USER_UID=${TORRUST_IDX_BACK_USER_UID:-1000} \
+    TORRUST_IDX_BACK_CONFIG=$(cat config-idx-back.mysql.local.toml) \
+    TORRUST_IDX_BACK_MYSQL_DATABASE="torrust_index_backend_e2e_testing" \
+    TORRUST_TRACKER_CONFIG=$(cat config-tracker.local.toml) \
+    TORRUST_TRACKER_API_TOKEN=${TORRUST_TRACKER_API_TOKEN:-MyAccessToken} \
+    docker compose up -d
+

docker/bin/run-e2e-tests.sh → docker/bin/e2e/run-e2e-tests.sh

@@ -35,14 +35,20 @@ wait_for_container_to_be_healthy() {
     return 1
 }
 
-# Install tool to create torrent files
+# Install tool to create torrent files.
+# It's needed by some tests to generate and parse test torrent files.
 cargo install imdl || exit 1
 
+# Install app (no docker) that will run the test suite against the E2E testing 
+# environment (in docker).
 cp .env.local .env || exit 1
 ./bin/install.sh || exit 1
 
+# TEST USING SQLITE
+echo "Running E2E tests using SQLite ..."
+
 # Start E2E testing environment
-./docker/bin/e2e-env-up.sh || exit 1
+./docker/bin/e2e/sqlite/e2e-env-up.sh || exit 1
 
 wait_for_container_to_be_healthy torrust-mysql-1 10 3
 # todo: implement healthchecks for tracker and backend and wait until they are healthy
@@ -54,7 +60,38 @@ sleep 20s
 docker ps
 
 # Run E2E tests with shared app instance
-TORRUST_IDX_BACK_E2E_SHARED=true cargo test || exit 1
+TORRUST_IDX_BACK_E2E_SHARED=true TORRUST_IDX_BACK_E2E_CONFIG_PATH="./config-idx-back.sqlite.local.toml" cargo test || exit 1
+
+# Stop E2E testing environment
+docker compose down
+
+# TEST USING MYSQL
+echo "Running E2E tests using MySQL ..."
+
+# Start E2E testing environment
+./docker/bin/e2e/mysql/e2e-env-up.sh || exit 1
+
+wait_for_container_to_be_healthy torrust-mysql-1 10 3
+# todo: implement healthchecks for tracker and backend and wait until they are healthy
+#wait_for_container torrust-tracker-1 10 3
+#wait_for_container torrust-idx-back-1 10 3
+sleep 20s
+
+# Just to make sure that everything is up and running
+docker ps
+
+# Database credentials
+MYSQL_USER="root"
+MYSQL_PASSWORD="root_secret_password"
+MYSQL_HOST="localhost"
+MYSQL_DATABASE="torrust_index_backend_e2e_testing"
+
+# Create the MySQL database for the index backend. Assumes MySQL client is installed.
+echo "Creating MySQL database $MYSQL_DATABASE for for E2E testing ..."
+mysql -h $MYSQL_HOST -u $MYSQL_USER -p$MYSQL_PASSWORD -e "CREATE DATABASE IF NOT EXISTS $MYSQL_DATABASE;"
+
+# Run E2E tests with shared app instance
+TORRUST_IDX_BACK_E2E_SHARED=true TORRUST_IDX_BACK_E2E_CONFIG_PATH="./config-idx-back.mysql.local.toml" cargo test || exit 1
 
 # Stop E2E testing environment
-./docker/bin/e2e-env-down.sh
+docker compose down

docker/bin/e2e-env-reset.sh → docker/bin/e2e/sqlite/e2e-env-reset.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
-# Delete the SQLite databases and recreate them.
+# Delete the databases and recreate them.
 
-./docker/bin/e2e-env-down.sh
+docker compose down
 
 rm -f ./storage/database/torrust_index_backend_e2e_testing.db
 rm -f ./storage/database/torrust_tracker_e2e_testing.db
@@ -23,4 +23,4 @@ if ! [ -f "./storage/database/torrust_tracker_e2e_testing.db" ]; then
     echo ";" | sqlite3 ./storage/database/torrust_tracker_e2e_testing.db
 fi
 
-./docker/bin/e2e-env-up.sh
+./docker/bin/e2e/sqlite/e2e-env-up.sh

docker/bin/e2e/sqlite/e2e-env-restart.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+docker compose down
+./docker/bin/e2e/sqlite/e2e-env-up.sh

docker/bin/e2e-env-up.sh → docker/bin/e2e/sqlite/e2e-env-up.sh

@@ -4,7 +4,7 @@ TORRUST_IDX_BACK_USER_UID=${TORRUST_IDX_BACK_USER_UID:-1000} \
     docker compose build
 
 TORRUST_IDX_BACK_USER_UID=${TORRUST_IDX_BACK_USER_UID:-1000} \
-    TORRUST_IDX_BACK_CONFIG=$(cat config-idx-back.local.toml) \
+    TORRUST_IDX_BACK_CONFIG=$(cat config-idx-back.sqlite.local.toml) \
     TORRUST_TRACKER_CONFIG=$(cat config-tracker.local.toml) \
     TORRUST_TRACKER_API_TOKEN=${TORRUST_TRACKER_API_TOKEN:-MyAccessToken} \
     docker compose up -d

src/databases/mysql.rs

@@ -250,7 +250,9 @@ impl Database for Mysql {
             .map(|v| i64::try_from(v.last_insert_id()).expect("last ID is larger than i64"))
             .map_err(|e| match e {
                 sqlx::Error::Database(err) => {
-                    if err.message().contains("UNIQUE") {
+                    if err.message().contains("Duplicate entry") {
+                        // Example error message when you try to insert a duplicate category:
+                        // Error: Duplicate entry 'category name SAMPLE_NAME' for key 'torrust_categories.name'
                         database::Error::CategoryAlreadyExists
                     } else {
                         database::Error::Error

tests/e2e/config.rs

@@ -1,7 +1,7 @@
 //! Initialize configuration for the shared E2E tests environment from a
 //! config file `config.toml` or env var.
 //!
-//! All environment variables are prefixed with `TORRUST_IDX_BACK_`.
+//! All environment variables are prefixed with `TORRUST_IDX_BACK_E2E`.
 use std::env;
 
 use torrust_index_backend::config::Configuration;
@@ -14,6 +14,9 @@ pub const ENV_VAR_E2E_SHARED: &str = "TORRUST_IDX_BACK_E2E_SHARED";
 /// The whole `config.toml` file content. It has priority over the config file.
 pub const ENV_VAR_E2E_CONFIG: &str = "TORRUST_IDX_BACK_E2E_CONFIG";
 
+/// The `config.toml` file location.
+pub const ENV_VAR_E2E_CONFIG_PATH: &str = "TORRUST_IDX_BACK_E2E_CONFIG_PATH";
+
 // Default values
 
 pub const ENV_VAR_E2E_DEFAULT_CONFIG_PATH: &str = "./config-idx-back.local.toml";
@@ -29,9 +32,11 @@ pub async fn init_shared_env_configuration() -> Configuration {
 
         Configuration::load_from_env_var(ENV_VAR_E2E_CONFIG).unwrap()
     } else {
-        println!("Loading configuration for E2E env from config file `{ENV_VAR_E2E_DEFAULT_CONFIG_PATH}`");
+        let config_path = env::var(ENV_VAR_E2E_CONFIG_PATH).unwrap_or_else(|_| ENV_VAR_E2E_DEFAULT_CONFIG_PATH.to_string());
+
+        println!("Loading configuration from config file `{config_path}`");
 
-        match Configuration::load_from_file(ENV_VAR_E2E_DEFAULT_CONFIG_PATH).await {
+        match Configuration::load_from_file(&config_path).await {
             Ok(config) => config,
             Err(error) => {
                 panic!("{}", error)

tests/e2e/environment.rs

@@ -1,5 +1,6 @@
 use std::env;
 
+use torrust_index_backend::databases::database;
 use torrust_index_backend::web::api::Version;
 
 use super::config::{init_shared_env_configuration, ENV_VAR_E2E_SHARED};
@@ -95,12 +96,53 @@ impl TestEnv {
         }
     }
 
-    /// Provides the database connect URL.
-    /// For example: `sqlite://storage/database/torrust_index_backend_e2e_testing.db?mode=rwc`.
+    /// Provides a database connect URL to connect to the database. For example:
+    ///
+    /// `sqlite://storage/database/torrust_index_backend_e2e_testing.db?mode=rwc`.
+    ///
+    /// It's used to run SQL queries against the database needed for some tests.
     pub fn database_connect_url(&self) -> Option<String> {
-        self.starting_settings
+        let internal_connect_url = self
+            .starting_settings
             .as_ref()
-            .map(|settings| settings.database.connect_url.clone())
+            .map(|settings| settings.database.connect_url.clone());
+
+        match self.state() {
+            State::RunningShared => {
+                if let Some(db_path) = internal_connect_url {
+                    let maybe_db_driver = database::get_driver(&db_path);
+
+                    return match maybe_db_driver {
+                        Ok(db_driver) => match db_driver {
+                            database::Driver::Sqlite3 => Some(db_path),
+                            database::Driver::Mysql => Some(Self::overwrite_mysql_host(&db_path, "localhost")),
+                        },
+                        Err(_) => None,
+                    };
+                }
+                None
+            }
+            State::RunningIsolated => internal_connect_url,
+            State::Stopped => None,
+        }
+    }
+
+    /// It overrides the "Host" in a `SQLx` database connection URL. For example:
+    ///
+    /// For:
+    ///
+    /// `mysql://root:root_secret_password@mysql:3306/torrust_index_backend_e2e_testing`.
+    ///
+    /// It changes the `mysql` host name to `localhost`:
+    ///
+    /// `mysql://root:root_secret_password@localhost:3306/torrust_index_backend_e2e_testing`.
+    ///
+    /// For E2E tests, we use docker compose, internally the backend connects to
+    /// the database using the "mysql" host, which is the docker compose service
+    /// name, but tests connects directly to the localhost since the `MySQL`
+    /// is exposed to the host.
+    fn overwrite_mysql_host(db_path: &str, new_host: &str) -> String {
+        db_path.replace("@mysql:", &format!("@{new_host}:"))
     }
 
     fn state(&self) -> State {