ci: refactor and improve speed and caching. add rustls flags to aws-r…

…ds (#1167) * consistency * ci: experiment new layout * cargo update * ci fix * cargo update * aws-rds macro use full paths * remove comments * clarify flags
shuttle-hq · Aug 22, 2023 · de362f8 · de362f8
1 parent 6ccf54c
commit de362f8
Show file tree

Hide file tree

Showing 9 changed files with 411 additions and 327 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/DEVELOPING.md b/DEVELOPING.md
@@ -136,8 +136,11 @@ In order to test local changes to the library crates, you may want to add the be
 
 ```toml
 [patch.crates-io]
-shuttle-service = { path = "[base]/shuttle/service" }
+shuttle-codegen = { path = "[base]/shuttle/codegen" }
+shuttle-common = { path = "[base]/shuttle/common" }
+shuttle-proto = { path = "[base]/shuttle/proto" }
 shuttle-runtime = { path = "[base]/shuttle/runtime" }
+shuttle-service = { path = "[base]/shuttle/service" }
 
 shuttle-aws-rds = { path = "[base]/shuttle/resources/aws-rds" }
 shuttle-persist = { path = "[base]/shuttle/resources/persist" }

diff --git a/cargo-shuttle/Cargo.toml b/cargo-shuttle/Cargo.toml
@@ -11,6 +11,7 @@ homepage = "https://www.shuttle.rs"
 anyhow = { workspace = true }
 async-trait = { workspace = true }
 bollard = { workspace = true }
+cargo-generate = "0.18.3"
 cargo_metadata = { workspace = true }
 chrono = { workspace = true }
 clap = { workspace = true, features = ["env"] }
@@ -34,12 +35,15 @@ reqwest = { workspace = true, features = ["json"] }
 reqwest-middleware = "0.2.0"
 reqwest-retry = "0.2.0"
 rmp-serde = { workspace = true }
+semver = "1.0.17"
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 strum = { workspace = true }
 tar = { workspace = true }
 tokio = { workspace = true, features = ["macros", "signal"] }
-tokio-tungstenite = { version = "0.19.0", features = ["rustls-tls-webpki-roots"] }
+tokio-tungstenite = { version = "0.19.0", features = [
+  "rustls-tls-webpki-roots",
+] }
 toml = { workspace = true }
 toml_edit = { workspace = true }
 tonic = { workspace = true }
@@ -52,8 +56,6 @@ tracing-subscriber = { workspace = true, features = [
 url = "2.3.1"
 uuid = { workspace = true, features = ["v4"] }
 webbrowser = "0.8.2"
-semver = "1.0.17"
-cargo-generate = "0.18.3"
 
 [dependencies.shuttle-common]
 workspace = true

diff --git a/deployer/prepare.sh b/deployer/prepare.sh
@@ -11,8 +11,11 @@ touch $CARGO_HOME/config.toml
 if [[ $PROD != "true" ]]; then
     echo '
     [patch.crates-io]
-    shuttle-service = { path = "/usr/src/shuttle/service" }
+    shuttle-codegen = { path = "/usr/src/shuttle/codegen" }
+    shuttle-common = { path = "/usr/src/shuttle/common" }
+    shuttle-proto = { path = "/usr/src/shuttle/proto" }
     shuttle-runtime = { path = "/usr/src/shuttle/runtime" }
+    shuttle-service = { path = "/usr/src/shuttle/service" }
 
     shuttle-aws-rds = { path = "/usr/src/shuttle/resources/aws-rds" }
     shuttle-persist = { path = "/usr/src/shuttle/resources/persist" }

diff --git a/e2e/tests/integration/helpers/mod.rs b/e2e/tests/integration/helpers/mod.rs
@@ -38,15 +38,18 @@ impl TempCargoHome {
                 write!(
                     config,
                     r#"[patch.crates-io]
-shuttle-service = {{ path = "{}" }}
+shuttle-codegen = {{ path = "{}" }}
+shuttle-common = {{ path = "{}" }}
+shuttle-proto = {{ path = "{}" }}
 shuttle-runtime = {{ path = "{}" }}
+shuttle-service = {{ path = "{}" }}
 
 shuttle-aws-rds = {{ path = "{}" }}
+shuttle-metadata = {{ path = "{}" }}
 shuttle-persist = {{ path = "{}" }}
 shuttle-shared-db = {{ path = "{}" }}
 shuttle-secrets = {{ path = "{}" }}
 shuttle-static-folder = {{ path = "{}" }}
-shuttle-metadata = {{ path = "{}" }}
 
 shuttle-axum = {{ path = "{}" }}
 shuttle-actix-web = {{ path = "{}" }}
@@ -60,20 +63,20 @@ shuttle-thruster = {{ path = "{}" }}
 shuttle-tide = {{ path = "{}" }}
 shuttle-tower = {{ path = "{}" }}
 shuttle-warp = {{ path = "{}" }}"#,
-                    WORKSPACE_ROOT.join("service").display(),
+                    WORKSPACE_ROOT.join("codegen").display(),
+                    WORKSPACE_ROOT.join("common").display(),
+                    WORKSPACE_ROOT.join("proto").display(),
                     WORKSPACE_ROOT.join("runtime").display(),
+                    WORKSPACE_ROOT.join("service").display(),
                     WORKSPACE_ROOT.join("resources").join("aws-rds").display(),
+                    WORKSPACE_ROOT.join("resources").join("metadata").display(),
                     WORKSPACE_ROOT.join("resources").join("persist").display(),
                     WORKSPACE_ROOT.join("resources").join("shared-db").display(),
                     WORKSPACE_ROOT.join("resources").join("secrets").display(),
                     WORKSPACE_ROOT
                         .join("resources")
                         .join("static-folder")
                         .display(),
-                    WORKSPACE_ROOT
-                        .join("resources")
-                        .join("metadata")
-                        .display(),
                     WORKSPACE_ROOT
                         .join("services")
                         .join("shuttle-axum")

diff --git a/resources/aws-rds/Cargo.toml b/resources/aws-rds/Cargo.toml
@@ -10,10 +10,13 @@ keywords = ["shuttle-service", "rds"]
 async-trait = "0.1.56"
 paste = "1.0.7"
 serde = { version = "1.0.148", features = ["derive"] }
-shuttle-service = { path = "../../service", version = "0.24.0" }
-sqlx = { version = "0.7.1", features = ["runtime-tokio-native-tls"] }
+shuttle-service = { path = "../../service", version = "0.24.0", default-features = false }
+sqlx = "0.7.1"
 
 [features]
-postgres = ["sqlx/postgres"]
-mysql = ["sqlx/mysql"]
-mariadb = ["sqlx/mysql"]
+postgres = ["sqlx/postgres", "sqlx/runtime-tokio-native-tls"]
+postgres-rustls = ["sqlx/postgres", "sqlx/runtime-tokio-rustls"]
+mysql = ["sqlx/mysql", "sqlx/runtime-tokio-native-tls"]
+mysql-rustls = ["sqlx/mysql", "sqlx/runtime-tokio-rustls"]
+mariadb = ["sqlx/mysql", "sqlx/runtime-tokio-native-tls"]
+mariadb-rustls = ["sqlx/mysql", "sqlx/runtime-tokio-rustls"]
diff --git a/resources/aws-rds/README.md b/resources/aws-rds/README.md
@@ -8,7 +8,10 @@ This plugin provisions databases on AWS RDS using [shuttle](https://www.shuttle.
 
 ## Usage
 
-Add `shuttle-aws-rds` to the dependencies for your service. Every engine is behind the following feature flags and attribute paths:
+Add `shuttle-aws-rds` to the dependencies for your service.
+Every engine is behind the following feature flags and attribute paths.
+The default is to use native TLS.
+You can also add `-rustls` after the feature flag, for example `postgres-rustls`.
 
 | Engine   | Feature flag | Attribute path            |
 |----------|--------------|---------------------------|

diff --git a/resources/aws-rds/src/lib.rs b/resources/aws-rds/src/lib.rs
@@ -1,32 +1,27 @@
 #![doc = include_str!("../README.md")]
 
-use async_trait::async_trait;
-use paste::paste;
-use serde::Serialize;
-use shuttle_service::{
-    database::{self, AwsRdsEngine},
-    error::CustomError,
-    DbInput, DbOutput, Factory, ResourceBuilder, Type,
-};
-
 macro_rules! aws_engine {
     ($feature:expr, $pool_path:path, $options_path:path, $struct_ident:ident) => {
-        paste! {
-            #[derive(Serialize)]
+        paste::paste! {
+            #[derive(serde::Serialize)]
             #[cfg(feature = $feature)]
             #[doc = "A resource connected to an AWS RDS " $struct_ident " instance"]
             pub struct $struct_ident{
-                config: DbInput,
+                config: shuttle_service::DbInput,
             }
 
             #[cfg(feature = $feature)]
             #[doc = "Gets a `sqlx::Pool` connected to an AWS RDS " $struct_ident " instance"]
-            #[async_trait]
-            impl ResourceBuilder<$pool_path> for $struct_ident {
-                const TYPE: Type = Type::Database(database::Type::AwsRds(AwsRdsEngine::$struct_ident));
+            #[async_trait::async_trait]
+            impl shuttle_service::ResourceBuilder<$pool_path> for $struct_ident {
+                const TYPE: shuttle_service::Type = shuttle_service::Type::Database(
+                    shuttle_service::database::Type::AwsRds(
+                        shuttle_service::database::AwsRdsEngine::$struct_ident
+                    )
+                );
 
-                type Config = DbInput;
-                type Output = DbOutput;
+                type Config = shuttle_service::DbInput;
+                type Output = shuttle_service::DbOutput;
 
                 fn new() -> Self {
                     Self { config: Default::default() }
@@ -36,20 +31,20 @@ macro_rules! aws_engine {
                     &self.config
                 }
 
-                async fn output(self, factory: &mut dyn Factory) -> Result<Self::Output, shuttle_service::Error> {
+                async fn output(self, factory: &mut dyn shuttle_service::Factory) -> Result<Self::Output, shuttle_service::Error> {
                     let info = match factory.get_environment() {
-                        shuttle_service::Environment::Production => DbOutput::Info(
+                        shuttle_service::Environment::Production => shuttle_service::DbOutput::Info(
                             factory
-                                .get_db_connection(database::Type::AwsRds(AwsRdsEngine::$struct_ident))
+                                .get_db_connection(shuttle_service::database::Type::AwsRds(shuttle_service::database::AwsRdsEngine::$struct_ident))
                                 .await?
                         ),
                         shuttle_service::Environment::Local => {
                             if let Some(local_uri) = self.config.local_uri {
-                                DbOutput::Local(local_uri)
+                                shuttle_service::DbOutput::Local(local_uri)
                             } else {
-                                DbOutput::Info(
+                                shuttle_service::DbOutput::Info(
                                     factory
-                                        .get_db_connection(database::Type::AwsRds(AwsRdsEngine::$struct_ident))
+                                        .get_db_connection(shuttle_service::database::Type::AwsRds(shuttle_service::database::AwsRdsEngine::$struct_ident))
                                         .await?
                                 )
                             }
@@ -61,16 +56,16 @@ macro_rules! aws_engine {
 
                 async fn build(build_data: &Self::Output) -> Result<$pool_path, shuttle_service::Error> {
                     let connection_string = match build_data {
-                        DbOutput::Local(local_uri) => local_uri.clone(),
-                        DbOutput::Info(info) => info.connection_string_private(),
+                        shuttle_service::DbOutput::Local(local_uri) => local_uri.clone(),
+                        shuttle_service::DbOutput::Info(info) => info.connection_string_private(),
                     };
 
                     let pool = $options_path::new()
                         .min_connections(1)
                         .max_connections(5)
                         .connect(&connection_string)
                         .await
-                        .map_err(CustomError::new)?;
+                        .map_err(shuttle_service::error::CustomError::new)?;
 
                     Ok(pool)
                 }