diff --git a/.circleci/config.yml b/.circleci/config.yml index 544eea807..967baf82f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ version: 2.1 -orbs: +orbs: docker-buildx: sensu/docker-buildx@1.1.1 aws-ecr: circleci/aws-ecr@8.1.2 win: circleci/windows@5.0 @@ -154,7 +154,7 @@ jobs: - install-protoc - run: cargo fmt --all --check - run: cargo install cargo-sort - # TODO: this is incompatible with workspace inheritance, uncomment when + # TODO: this is incompatible with workspace inheritance, uncomment when # https://github.com/DevinR528/cargo-sort/pull/29 is merged # - run: cargo sort --check --workspace - run: cargo check --workspace --all-targets @@ -191,7 +191,7 @@ jobs: - apply-patches - run: cargo fmt --all --check --manifest-path << parameters.path >>/Cargo.toml - run: cargo install cargo-sort - # TODO: this is incompatible with workspace inheritance, uncomment when + # TODO: this is incompatible with workspace inheritance, uncomment when # https://github.com/DevinR528/cargo-sort/pull/29 is merged # - run: cargo sort --check << parameters.path >> - run: | @@ -388,79 +388,101 @@ jobs: workflows: ci: jobs: - - workspace-fmt - - workspace-clippy: - name: workspace-clippy-<< matrix.framework >> - requires: - - workspace-fmt - matrix: - parameters: - framework: ["web-actix-web", "web-axum", "web-rocket", "web-poem", "web-thruster", "web-tide", "web-tower","web-warp", "web-salvo", "bot-serenity", "bot-poise"] - - check-standalone: - matrix: - parameters: - path: - - resources/aws-rds - - resources/persist - - resources/secrets - - resources/shared-db - - resources/static-folder - - service-test: - requires: - - workspace-clippy - - platform-test: - requires: - - workspace-clippy - matrix: - parameters: - crate: ["shuttle-auth", "shuttle-deployer", "cargo-shuttle", "shuttle-codegen", "shuttle-common", "shuttle-proto", "shuttle-provisioner"] - - e2e-test: - requires: - - service-test - - platform-test - - check-standalone - filters: - branches: - only: production - - build-and-push: - requires: - - e2e-test - filters: - branches: - only: production - - build-binaries-linux: - name: build-binaries-x86_64-gnu - image: ubuntu-2204:2022.04.1 - target: x86_64-unknown-linux-gnu - resource_class: medium - filters: - branches: - only: production - - build-binaries-linux: - name: build-binaries-x86_64-musl - image: ubuntu-2204:2022.04.1 - target: x86_64-unknown-linux-musl - resource_class: medium - filters: - branches: - only: production - - build-binaries-linux: - name: build-binaries-aarch64 - image: ubuntu-2004:202101-01 - target: aarch64-unknown-linux-musl - resource_class: arm.medium - filters: - branches: - only: production - - build-binaries-windows: + - workspace-fmt + - workspace-clippy: + name: workspace-clippy-<< matrix.framework >> + requires: + - workspace-fmt + matrix: + parameters: + framework: + [ + "web-actix-web", + "web-axum", + "web-rocket", + "web-poem", + "web-thruster", + "web-tide", + "web-tower", + "web-warp", + "web-salvo", + "bot-serenity", + "bot-poise", + ] + - check-standalone: + matrix: + parameters: + path: + - resources/aws-rds + - resources/persist + - resources/secrets + - resources/shared-db + - resources/static-folder + - service-test: + requires: + - workspace-clippy + - platform-test: + requires: + - workspace-clippy + matrix: + parameters: + crate: + [ + "shuttle-auth", + "shuttle-deployer", + "cargo-shuttle", + "shuttle-codegen", + "shuttle-common", + "shuttle-proto", + "shuttle-provisioner", + ] + - e2e-test: + requires: + - service-test + - platform-test + - check-standalone + filters: + branches: + only: production + - build-and-push: + requires: + - e2e-test + filters: + branches: + only: production + - build-binaries-linux: + name: build-binaries-x86_64-gnu + image: ubuntu-2204:2022.04.1 + target: x86_64-unknown-linux-gnu + resource_class: medium filters: branches: only: production - - build-binaries-mac: + - build-binaries-linux: + name: build-binaries-x86_64-musl + image: ubuntu-2204:2022.04.1 + target: x86_64-unknown-linux-musl + resource_class: medium filters: branches: - only: production - - publish-github-release: + only: production + - build-binaries-linux: + name: build-binaries-aarch64 + image: ubuntu-2004:202101-01 + target: aarch64-unknown-linux-musl + resource_class: arm.medium + filters: + branches: + only: production + - build-binaries-windows: + filters: + branches: + only: production + - build-binaries-mac: + filters: + branches: + only: production + - publish-github-release: requires: - build-binaries-x86_64-gnu - build-binaries-x86_64-musl @@ -469,4 +491,4 @@ workflows: - build-binaries-mac filters: branches: - only: production + only: production diff --git a/.github/ISSUE_TEMPLATE/BUG-REPORT.yml b/.github/ISSUE_TEMPLATE/BUG-REPORT.yml new file mode 100644 index 000000000..69bfdb13b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/BUG-REPORT.yml @@ -0,0 +1,61 @@ +name: Bug Report +description: File a bug report +title: "[Bug]: " +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + - type: textarea + id: what-happened + attributes: + label: What happened? + description: Also tell us, what did you expect to happen? + placeholder: Tell us what you see! + value: "A bug happened!" + validations: + required: true + - type: input + id: version + attributes: + label: Version + description: What version of `cargo-shuttle` are you running (`cargo shuttle --version`)? + placeholder: "v0.11.0" + validations: + required: true + - type: dropdown + id: os + attributes: + label: Which operating systems are you seeing the problem on? + multiple: true + options: + - macOS + - Windows + - Linux + validations: + required: true + - type: dropdown + id: architecture + attributes: + label: Which CPU architectures are you seeing the problem on? + multiple: true + options: + - x86_64 + - ARM64 + - Other + - type: textarea + id: logs + attributes: + label: Relevant log output + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + render: sh + - type: checkboxes + id: duplicate + attributes: + label: Duplicate declaration + description: Please confirm that you are not creating a duplicate issue. + options: + - label: I have searched the issues and there are none like this. + required: true + diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..3455e4136 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: true +contact_links: + - name: Discord + url: https://discord.gg/shuttle + about: Feel free to reach out on our Discord should you have any questions! diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..86bbe92d9 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,9 @@ +## Description of change + +Please write a summary of your changes and why you made them. + +Be sure to reference any related issues by adding `closes issue #`. + +## How Has This Been Tested (if applicable)? + +Please describe the tests that you ran to verify your changes. diff --git a/.gitignore b/.gitignore index 4b15dd269..4b9ed654c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,8 @@ # Generated by Cargo # will have compiled files and executables **/target/ -Cargo.lock +# Ignore the cargo lockfiles not in the workspace root +*/**/Cargo.lock # These are backup files generated by rustfmt **/*.rs.bk diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8aab90e89..51c8a8511 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,22 +3,23 @@ ## Raise an Issue Raising [issues](https://github.com/shuttle-hq/shuttle/issues) is encouraged. + ## Docs If you found an error in our docs, or you simply want to make them better, contributions to our [docs](https://github.com/shuttle-hq/shuttle-docs) are always appreciated! ## Running Locally + You can use Docker and docker-compose to test shuttle locally during development. See the [Docker install](https://docs.docker.com/get-docker/) and [docker-compose install](https://docs.docker.com/compose/install/) instructions if you do not have them installed already. > Note for Windows: The current [Makefile](https://github.com/shuttle-hq/shuttle/blob/main/Makefile) does not work on Windows systems by itself - if you want to build the local environment on Windows you could use [Windows Subsystem for Linux](https://learn.microsoft.com/en-us/windows/wsl/install). Additional Windows considerations are listed at the bottom of this page. - > Note for Linux: When building on Linux systems, if the error unknown flag: --build-arg is received, install the docker-buildx package using the package management tool for your particular system. Clone the shuttle repository (or your fork): -``` +```bash git clone git@github.com:shuttle-hq/shuttle.git cd shuttle ``` @@ -28,9 +29,11 @@ You should now be ready to setup a local environment to test code changes to cor From the root of the shuttle repo, build the required images with: ```bash -make images +USE_PANAMAX=disable make images ``` +> Note: The stack uses [panamax](https://github.com/panamax-rs/panamax) by default to mirror crates.io content. We do this in order to avoid overloading upstream mirrors and hitting rate limits. After syncing the cache, expect to see the panamax volume take about 100GiB of space. This may not be desirable for local testing. To avoid using panamax, run `USE_PANAMAX=disable make images` instead. + The images get built with [cargo-chef](https://github.com/LukeMathWalker/cargo-chef) and therefore support incremental builds (most of the time). So they will be much faster to re-build after an incremental change in your code - should you wish to deploy it locally straight away. You can now start a local deployment of shuttle and the required containers with: @@ -39,13 +42,15 @@ You can now start a local deployment of shuttle and the required containers with make up ``` +> Note: `make up` does not start [panamax](https://github.com/panamax-rs/panamax) by default, if you do need to start panamax for local development, run this command with `make COMPOSE_PROFILES=panamax up`. + > Note: Other useful commands can be found within the [Makefile](https://github.com/shuttle-hq/shuttle/blob/main/Makefile). The API is now accessible on `localhost:8000` (for app proxies) and `localhost:8001` (for the control plane). When running `cargo run --bin cargo-shuttle` (in a debug build), the CLI will point itself to `localhost` for its API calls. In order to test local changes to the library crates, you may want to add the below to a `.cargo/config.toml` file. (See [Overriding Dependencies](https://doc.rust-lang.org/cargo/reference/overriding-dependencies.html) for more) -``` toml +```toml [patch.crates-io] shuttle-service = { path = "[base]/shuttle/service" } shuttle-common = { path = "[base]/shuttle/common" } @@ -58,7 +63,7 @@ shuttle-static-folder = { path = "[base]/shuttle/resources/static-folder" } ``` Before we can login to our local instance of shuttle, we need to create a user. -The following command inserts a user into the gateway state with admin privileges: +The following command inserts a user into the `auth` state with admin privileges: ```bash docker compose --file docker-compose.rendered.yml --project-name shuttle-dev exec auth /usr/local/bin/service --state=/var/lib/shuttle-auth init --name admin --key test-key @@ -182,6 +187,7 @@ We will squash commits before merging to main. If you do want to squash commits, after the review process has started, the commit history can be useful for reviewers. Before committing: + - Make sure your commits don't trigger any warnings from Clippy by running: `cargo clippy --tests --all-targets`. If you have a good reason to contradict Clippy, insert an `#[allow(clippy::)]` macro, so that it won't complain. - Make sure your code is correctly formatted: `cargo fmt --all --check`. - Make sure your `Cargo.toml`'s are sorted: `cargo +nightly sort --workspace`. This command uses the [cargo-sort crate](https://crates.io/crates/cargo-sort) to sort the `Cargo.toml` dependencies alphabetically. @@ -227,6 +233,7 @@ graph BT First, `provisioner`, `gateway`, `deployer`, and `cargo-shuttle` are binary crates with `provisioner`, `gateway` and `deployer` being backend services. The `cargo-shuttle` binary is the `cargo shuttle` command used by users. The rest are the following libraries: + - `common` contains shared models and functions used by the other libraries and binaries. - `codegen` contains our proc-macro code which gets exposed to user services from `service` by the `codegen` feature flag. The redirect through `service` is to make it available under the prettier name of `shuttle_service::main`. - `service` is where our special `Service` trait is defined. Anything implementing this `Service` can be loaded by the `deployer` and the local runner in `cargo-shuttle`. @@ -238,20 +245,22 @@ Lastly, the `user service` is not a folder in this repository, but is the user s ## Windows Considerations -Currently, if you try to use 'make images' on Windows, you may find that the shell files cannot be read by Bash/WSL. This is due to the fact that Windows may have pulled the files in CRLF format rather than LF[^1], which causes problems with Bash as to run the commands, Linux needs the file in LF format. +Currently, if you try to use 'make images' on Windows, you may find that the shell files cannot be read by Bash/WSL. This is due to the fact that Windows may have pulled the files in CRLF format rather than LF[^1], which causes problems with Bash as to run the commands, Linux needs the file in LF format. Thankfully, we can fix this problem by simply using the `git config core.autocrlf` command to change how Git handles line endings. It takes a single argument: -``` +```bash git config --global core.autocrlf input ``` This should allow you to run `make images` and other Make commands with no issues. If you need to change it back for whatever reason, you can just change the last argument from 'input' to 'true' like so: -``` + +```bash git config --global core.autocrlf true ``` + After you run this command, you should be able to checkout projects that are maintained using CRLF (Windows) again. [^1]: https://git-scm.com/book/en/v2/Customizing-Git-Git-Configuration#_core_autocrlf diff --git a/Cargo.lock b/Cargo.lock index 495a5ef57..e15c70b8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1262,9 +1262,9 @@ dependencies = [ [[package]] name = "bson" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d76085681585d39016f4d3841eb019201fc54d2dd0d92ad1e4fab3bfb32754" +checksum = "8746d07211bb12a7c34d995539b4a2acd4e0b0e757de98ce2ab99bcf17443fad" dependencies = [ "ahash", "base64 0.13.1", @@ -1489,7 +1489,7 @@ dependencies = [ [[package]] name = "cargo-shuttle" -version = "0.11.0" +version = "0.11.2" dependencies = [ "anyhow", "assert_cmd", @@ -3888,9 +3888,9 @@ dependencies = [ [[package]] name = "mongodb" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a1df476ac9541b0e4fdc8e2cc48884e66c92c933cd17a1fd75e68caf75752e" +checksum = "a37fe10c1485a0cd603468e284a1a8535b4ecf46808f5f7de3639a1e1252dbf8" dependencies = [ "async-trait", "base64 0.13.1", @@ -3898,20 +3898,21 @@ dependencies = [ "bson", "chrono", "derivative", + "derive_more", "futures-core", "futures-executor", + "futures-io", "futures-util", "hex 0.4.3", "hmac 0.12.1", "lazy_static", "md-5", - "os_info", "pbkdf2", "percent-encoding", "rand 0.8.5", "rustc_version_runtime", "rustls", - "rustls-pemfile 0.3.0", + "rustls-pemfile 1.0.1", "serde", "serde_bytes", "serde_with", @@ -3928,7 +3929,7 @@ dependencies = [ "trust-dns-proto", "trust-dns-resolver", "typed-builder", - "uuid 0.8.2", + "uuid 1.2.2", "webpki-roots", ] @@ -4390,9 +4391,9 @@ checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" [[package]] name = "pbkdf2" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271779f35b581956db91a3e55737327a03aa051e90b1c47aeb189508533adfd7" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" dependencies = [ "digest 0.10.3", ] @@ -5470,15 +5471,6 @@ dependencies = [ "base64 0.13.1", ] -[[package]] -name = "rustls-pemfile" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee86d63972a7c661d1536fefe8c3c8407321c3df668891286de28abcd087360" -dependencies = [ - "base64 0.13.1", -] - [[package]] name = "rustls-pemfile" version = "1.0.1" @@ -5966,7 +5958,7 @@ dependencies = [ [[package]] name = "shuttle-common" -version = "0.11.0" +version = "0.11.2" dependencies = [ "anyhow", "async-trait", @@ -6007,7 +5999,7 @@ dependencies = [ [[package]] name = "shuttle-deployer" -version = "0.11.0" +version = "0.11.1" dependencies = [ "anyhow", "async-trait", @@ -6053,7 +6045,7 @@ dependencies = [ [[package]] name = "shuttle-gateway" -version = "0.11.0" +version = "0.11.2" dependencies = [ "acme2", "anyhow", @@ -7040,6 +7032,7 @@ checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" dependencies = [ "bytes 1.3.0", "futures-core", + "futures-io", "futures-sink", "pin-project-lite 0.2.9", "tokio", @@ -7599,15 +7592,6 @@ dependencies = [ "rand 0.6.5", ] -[[package]] -name = "uuid" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" -dependencies = [ - "getrandom 0.2.7", -] - [[package]] name = "uuid" version = "1.2.2" @@ -7862,9 +7846,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.3" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d8de8415c823c8abd270ad483c6feeac771fad964890779f9a8cb24fbbc1bf" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" dependencies = [ "webpki", ] diff --git a/Cargo.toml b/Cargo.toml index 01efa6a18..5bd66069a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,14 +31,14 @@ repository = "https://github.com/shuttle-hq/shuttle" # https://doc.rust-lang.org/cargo/reference/workspaces.html#the-workspacedependencies-table [workspace.dependencies] shuttle-codegen = { path = "codegen", version = "0.11.0" } -shuttle-common = { path = "common", version = "0.11.0" } +shuttle-common = { path = "common", version = "0.11.2" } shuttle-proto = { path = "proto", version = "0.11.0" } shuttle-service = { path = "service", version = "0.11.0" } anyhow = "1.0.66" async-trait = "0.1.58" axum = "0.6.0" -chrono = "0.4.23" +chrono = { version = "0.4.23", default-features = false, features = ["clock"] } clap = { version = "4.0.27", features = [ "derive" ] } headers = "0.3.8" http = "0.2.8" @@ -48,12 +48,12 @@ once_cell = "1.16.0" opentelemetry = { version = "0.18.0", features = ["rt-tokio"] } opentelemetry-http = "0.7.0" pin-project = "1.0.12" +portpicker = "0.1.1" rand = "0.8.5" ring = "0.16.20" serde = "1.0.148" serde_json = "1.0.89" strum = { version = "0.24.1", features = ["derive"] } -portpicker = "0.1.1" thiserror = "1.0.37" tower = "0.4.13" tower-http = { version = "0.3.4", features = ["trace"] } diff --git a/Containerfile b/Containerfile index c2a921825..0765cba36 100644 --- a/Containerfile +++ b/Containerfile @@ -37,8 +37,9 @@ COPY --from=cache /build/ /usr/src/shuttle/ FROM shuttle-common ARG folder +ARG prepare_args COPY ${folder}/prepare.sh /prepare.sh -RUN /prepare.sh +RUN /prepare.sh "${prepare_args}" ARG CARGO_PROFILE COPY --from=builder /build/target/${CARGO_PROFILE}/shuttle-${folder} /usr/local/bin/service ARG RUSTUP_TOOLCHAIN diff --git a/Makefile b/Makefile index 21914265d..2bed9be28 100644 --- a/Makefile +++ b/Makefile @@ -67,12 +67,18 @@ POSTGRES_EXTRA_PATH?=./extras/postgres POSTGRES_TAG?=14 PANAMAX_EXTRA_PATH?=./extras/panamax -PANAMAX_TAG?=1.0.6 +PANAMAX_TAG?=1.0.12 OTEL_EXTRA_PATH?=./extras/otel OTEL_TAG?=0.72.0 -DOCKER_COMPOSE_ENV=STACK=$(STACK) BACKEND_TAG=$(BACKEND_TAG) DEPLOYER_TAG=$(DEPLOYER_TAG) PROVISIONER_TAG=$(PROVISIONER_TAG) POSTGRES_TAG=${POSTGRES_TAG} PANAMAX_TAG=${PANAMAX_TAG} OTEL_TAG=${OTEL_TAG} APPS_FQDN=$(APPS_FQDN) DB_FQDN=$(DB_FQDN) POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) RUST_LOG=$(RUST_LOG) CONTAINER_REGISTRY=$(CONTAINER_REGISTRY) MONGO_INITDB_ROOT_USERNAME=$(MONGO_INITDB_ROOT_USERNAME) MONGO_INITDB_ROOT_PASSWORD=$(MONGO_INITDB_ROOT_PASSWORD) DD_ENV=$(DD_ENV) USE_TLS=$(USE_TLS) +USE_PANAMAX?=enable +ifeq ($(USE_PANAMAX), enable) +PREPARE_ARGS+=-p +COMPOSE_PROFILES+=panamax +endif + +DOCKER_COMPOSE_ENV=STACK=$(STACK) BACKEND_TAG=$(BACKEND_TAG) DEPLOYER_TAG=$(DEPLOYER_TAG) PROVISIONER_TAG=$(PROVISIONER_TAG) POSTGRES_TAG=${POSTGRES_TAG} PANAMAX_TAG=${PANAMAX_TAG} OTEL_TAG=${OTEL_TAG} APPS_FQDN=$(APPS_FQDN) DB_FQDN=$(DB_FQDN) POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) RUST_LOG=$(RUST_LOG) CONTAINER_REGISTRY=$(CONTAINER_REGISTRY) MONGO_INITDB_ROOT_USERNAME=$(MONGO_INITDB_ROOT_USERNAME) MONGO_INITDB_ROOT_PASSWORD=$(MONGO_INITDB_ROOT_PASSWORD) DD_ENV=$(DD_ENV) USE_TLS=$(USE_TLS) COMPOSE_PROFILES=$(COMPOSE_PROFILES) .PHONY: images clean src up down deploy shuttle-% postgres docker-compose.rendered.yml test bump-% deploy-examples publish publish-% --validate-version @@ -91,12 +97,14 @@ postgres: $(POSTGRES_EXTRA_PATH) panamax: - docker buildx build \ - --build-arg PANAMAX_TAG=$(PANAMAX_TAG) \ - --tag $(CONTAINER_REGISTRY)/panamax:$(PANAMAX_TAG) \ - $(BUILDX_FLAGS) \ - -f $(PANAMAX_EXTRA_PATH)/Containerfile \ - $(PANAMAX_EXTRA_PATH) + if [ $(USE_PANAMAX) = "enable" ]; then \ + docker buildx build \ + --build-arg PANAMAX_TAG=$(PANAMAX_TAG) \ + --tag $(CONTAINER_REGISTRY)/panamax:$(PANAMAX_TAG) \ + $(BUILDX_FLAGS) \ + -f $(PANAMAX_EXTRA_PATH)/Containerfile \ + $(PANAMAX_EXTRA_PATH); \ + fi otel: docker buildx build \ @@ -115,6 +123,9 @@ deploy: docker-compose.yml test: cd e2e; POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) APPS_FQDN=$(APPS_FQDN) cargo test $(CARGO_TEST_FLAGS) -- --nocapture +# Start the containers locally. This does not start panamax by default, +# to start panamax locally run this command with the COMPOSE_PROFILES=panamax +# environment variable. up: docker-compose.rendered.yml CONTAINER_REGISTRY=$(CONTAINER_REGISTRY) $(DOCKER_COMPOSE) -f $< -p $(STACK) up -d $(DOCKER_COMPOSE_FLAGS) @@ -124,7 +135,8 @@ down: docker-compose.rendered.yml shuttle-%: ${SRC} Cargo.lock docker buildx build \ --build-arg folder=$(*) \ - --build-arg RUSTUP_TOOLCHAIN=$(RUSTUP_TOOLCHAIN) \ + --build-arg prepare_args=$(PREPARE_ARGS) \ + --build-arg RUSTUP_TOOLCHAIN=$(RUSTUP_TOOLCHAIN) \ --build-arg CARGO_PROFILE=$(CARGO_PROFILE) \ --tag $(CONTAINER_REGISTRY)/$(*):$(COMMIT_SHA) \ --tag $(CONTAINER_REGISTRY)/$(*):$(TAG) \ diff --git a/README.md b/README.md index 2035ef09b..03b30c12b 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ +


- docs + docs language @@ -16,6 +17,7 @@ discord

+ --- @@ -24,9 +26,10 @@ [Shuttle](https://www.shuttle.rs/) is a Rust-native cloud development platform that lets you deploy your Rust apps for free. Shuttle is built for productivity, reliability and performance: + - Zero-Configuration support for Rust using annotations - Automatic resource provisioning (databases, caches, subdomains, etc.) via [Infrastructure-From-Code](https://www.shuttle.rs/blog/2022/05/09/ifc) -- First-class support for popular Rust frameworks ([Actix](https://docs.shuttle.rs/examples/actix), [Rocket](https://docs.shuttle.rs/examples/rocket), [Axum](https://docs.shuttle.rs/examples/axum), +- First-class support for popular Rust frameworks ([Actix](https://docs.shuttle.rs/examples/actix), [Rocket](https://docs.shuttle.rs/examples/rocket), [Axum](https://docs.shuttle.rs/examples/axum), [Tide](https://docs.shuttle.rs/examples/tide), [Poem](https://docs.shuttle.rs/examples/poem) and [Tower](https://docs.shuttle.rs/examples/tower)) - Support for deploying Discord bots using [Serenity](https://docs.shuttle.rs/examples/serenity) - Scalable hosting (with optional self-hosting) @@ -52,10 +55,13 @@ cargo shuttle login ``` To initialize your project, simply write: + ```bash cargo shuttle init --axum hello-world ``` + And to deploy it, write: + ```bash cargo shuttle project new cargo shuttle project status // until the project is "ready" @@ -75,14 +81,15 @@ $ cargo shuttle deploy Created At: 2022-04-01 08:32:34.412602556 UTC ``` -Feel free to build on-top of the generated `hello-world` boilerplate or take a stab at one of our [examples](https://docs.shuttle.rs/guide/axum-examples.html#hello-world). +Feel free to build on-top of the generated `hello-world` boilerplate or take a stab at one of our [examples](https://docs.shuttle.rs/examples/axum). For the full documentation, visit [our docs](https://docs.shuttle.rs). + ## Contributing to shuttle Contributing to shuttle is highly encouraged! -If you want to setup a local environment to test code changes to core `shuttle` packages, or want to contribute to the project check out [our docs](https://docs.shuttle.rs/community/contribute). +If you want to setup a local environment to test code changes to core `shuttle` packages, or want to contribute to the project check out [our docs](https://docs.shuttle.rs/community/contribute). Even if you are not planning to submit any code; joining our [Discord server](https://discord.gg/shuttle) and providing feedback helps us a lot! @@ -102,12 +109,12 @@ If you have any requests or suggestions feel free to open an issue. ## Status - [x] Alpha: We are testing Shuttle, API and deployments may be unstable -- [x] Public Alpha: Anyone can sign up, but go easy on us, +- [x] Public Alpha: Anyone can sign up, but go easy on us, there are a few kinks - [ ] Public Beta: Stable enough for most non-enterprise use-cases - [ ] Public: Production-ready! -We are currently in Public Alpha. Watch "releases" of this repo to get +We are currently in Public Alpha. Watch "releases" of this repo to get notified of major updates! ## Contributors ✨ diff --git a/admin/README.md b/admin/README.md index 6d56d6266..12e6e0ccc 100644 --- a/admin/README.md +++ b/admin/README.md @@ -1,6 +1,10 @@ -_Small utility used by the shuttle admin for common tasks_ +# Admin + + +*Small utility used by the shuttle admin for common tasks* ## How to test custom domain certificates locally + For local testing it is easiest to use the [Pebble](https://github.com/letsencrypt/pebble) server. So install it using whatever method works for your system. It is included in the nix environment if you use it though. @@ -31,4 +35,4 @@ cargo run -p shuttle-admin -- --api-url http://localhost:8001 acme create-accoun Safe the account JSON in a local file and use it to test creating new certificate. However, you'll the FQDN you're using for testnig to resolve to your local machine. So create an `A` record for it on your DNS with the value -`127.0.0.1`. And Bob's your uncle πŸŽ‰ +`127.0.0.1`. And Bob's your uncle πŸŽ‰ diff --git a/cargo-shuttle/Cargo.toml b/cargo-shuttle/Cargo.toml index 6173511dc..64d9d8f26 100644 --- a/cargo-shuttle/Cargo.toml +++ b/cargo-shuttle/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cargo-shuttle" -version = "0.11.0" +version = "0.11.2" edition.workspace = true license.workspace = true repository.workspace = true diff --git a/cargo-shuttle/README.md b/cargo-shuttle/README.md index afac19853..ae1da5f10 100644 --- a/cargo-shuttle/README.md +++ b/cargo-shuttle/README.md @@ -1,18 +1,24 @@ +
# cargo-shuttle

+ + docs + language - - build status + + build status discord

+ + `cargo-shuttle` is your commandline tool for deploying web apps on [shuttle](https://www.shuttle.rs/), the stateful serverless web platform for Rust. @@ -26,62 +32,65 @@ --- - -

Installation

-
+ +

Installation

`cargo-shuttle` is available for macOS, Linux, and Windows. To install the commandline tool, run: -```sh -$ cargo install cargo-shuttle +```bash +cargo install cargo-shuttle ``` --- - -

Subcommands

-
+ +

Subcommands

`cargo-shuttle`'s subcommands help you build and deploy web apps from start to finish. -Run `cargo-shuttle --help` to see the basic usage: - -``` -USAGE: - cargo-shuttle [OPTIONS] - -OPTIONS: - --api-url run this command against the api at the supplied url (allows targeting a custom deployed instance for this command only) [env: SHUTTLE_API=] - -h, --help Print help information - --name Specify the name of the project (overrides crate name) - -V, --version Print version information - --working-directory Specify the working directory [default: .] - -SUBCOMMANDS: - auth create user credentials for the shuttle platform - delete delete this shuttle service - deploy deploy a shuttle service - deployment manage deployments of a shuttle service - help Print this message or the help of the given subcommand(s) - init create a new shuttle service - login login to the shuttle platform - logs view the logs of a deployment in this shuttle service - run run a shuttle service locally - status view the status of a shuttle service +Run `cargo shuttle help` to see the basic usage: + +```text +Usage: cargo-shuttle [OPTIONS] + +Commands: + deploy deploy a shuttle service + deployment manage deployments of a shuttle service + init create a new shuttle service + generate generate shell completions + status view the status of a shuttle service + logs view the logs of a deployment in this shuttle service + clean remove artifacts that were generated by cargo + stop stop this shuttle service + secrets manage secrets for this shuttle service + login login to the shuttle platform + logout log out of the shuttle platform + run run a shuttle service locally + feedback Open an issue on github and provide feedback + project manage a project on shuttle + help Print this message or the help of the given subcommand(s) + +Options: + --api-url run this command against the api at the supplied url (allows targeting a custom deployed instance for this command only) [env: SHUTTLE_API=] + --working-directory Specify the working directory [default: .] + --name Specify the name of the project (overrides crate name) + -h, --help Print help + -V, --version Print version ``` ### Subcommand: `init` -To initialize a shuttle project with boilerplates, run `cargo shuttle init [OPTIONS] [PATH]`. +To initialize a shuttle project with boilerplates, run `cargo shuttle init [OPTIONS] [PATH]`. Currently, `cargo shuttle init` supports the following frameworks: -- `--axum`: for [axum](https://github.com/tokio-rs/axum) framework - `--actix-web`: for [actix web](https://actix.rs/) framework +- `--axum`: for [axum](https://github.com/tokio-rs/axum) framework - `--poem`: for [poem](https://github.com/poem-web/poem) framework +- `--poise`: for [poise](https://github.com/serenity-rs/poise) discord bot framework - `--rocket`: for [rocket](https://rocket.rs/) framework - `--salvo`: for [salvo](https://salvo.rs/) framework -- `--serenity`: for [serenity](https://serenity.rs/) discord bot framework +- `--serenity`: for [serenity](https://github.com/serenity-rs/serenity) discord bot framework - `--thruster`: for [thruster](https://github.com/thruster-rs/Thruster) framework - `--tide`: for [tide](https://github.com/http-rs/tide) framework - `--tower`: for [tower](https://github.com/tower-rs/tower) library @@ -90,10 +99,11 @@ Currently, `cargo shuttle init` supports the following frameworks: For example, running the following command will initialize a project for [rocket](https://rocket.rs/): ```sh -$ cargo shuttle init --rocket my-rocket-app +cargo shuttle init --rocket my-rocket-app ``` This should generate the following dependency in `Cargo.toml`: + ```toml shuttle-service = { version = "0.11.0", features = ["web-rocket"] } ``` @@ -104,16 +114,14 @@ The following boilerplate code should be generated into `src/lib.rs`: #[macro_use] extern crate rocket; -use shuttle_service::ShuttleRocket; - -#[get("/hello")] -fn hello() -> &'static str { +#[get("/")] +fn index() -> &'static str { "Hello, world!" } #[shuttle_service::main] -async fn init() -> ShuttleRocket { - let rocket = rocket::build().mount("/", routes![hello]); +async fn rocket() -> shuttle_service::ShuttleRocket { + let rocket = rocket::build().mount("/hello", routes![index]); Ok(rocket) } @@ -125,7 +133,7 @@ To run the shuttle project locally, use the following command: ```sh # Inside your shuttle project -$ cargo shuttle run +cargo shuttle run ``` This will compile your shuttle project and start it on the default port `8000`. Test it by: @@ -141,13 +149,13 @@ Use `cargo shuttle login` inside your shuttle project to generate an API key for ```sh # Inside a shuttle project -$ cargo shuttle login +cargo shuttle login ``` This should automatically open a browser window with an auto-generated API key for your project. Simply copy-paste the API key back in your terminal or run the following command to complete login: ```sh -$ cargo shuttle login --api-key your-api-key-from-browser +cargo shuttle login --api-key ``` ### Subcommand: `deploy` @@ -155,8 +163,8 @@ $ cargo shuttle login --api-key your-api-key-from-browser To deploy your shuttle project to the cloud, run: ```sh -$ cargo shuttle project new -$ cargo shuttle deploy +cargo shuttle project new +cargo shuttle deploy ``` Your service will immediately be available at `{crate_name}.shuttleapp.rs`. For instance: @@ -171,7 +179,7 @@ Hello, world! Check the status of your deployed shuttle project with: ```sh -$ cargo shuttle status +cargo shuttle status ``` ### Subcommand: `logs` @@ -179,22 +187,21 @@ $ cargo shuttle status Check the logs of your deployed shuttle project with: ```sh -$ cargo shuttle logs +cargo shuttle logs ``` -### Subcommand: `delete` +### Subcommand: `stop` -Once you are done with a deployment, you can delete it by running: +Once you are done with a deployment, you can stop it by running: ```sh -$ cargo shuttle delete +cargo shuttle stop ``` --- - -

Development

-
+ +

Development

Thanks for using `cargo-shuttle`! We’re very happy to have you with us! diff --git a/cargo-shuttle/src/args.rs b/cargo-shuttle/src/args.rs index 2cbc8e02c..fb1347198 100644 --- a/cargo-shuttle/src/args.rs +++ b/cargo-shuttle/src/args.rs @@ -8,7 +8,7 @@ use std::{ use clap::builder::{OsStringValueParser, PossibleValue, TypedValueParser}; use clap::Parser; use clap_complete::Shell; -use shuttle_common::project::ProjectName; +use shuttle_common::{models::project::IDLE_MINUTES, project::ProjectName}; use uuid::Uuid; use crate::init::Framework; @@ -108,7 +108,11 @@ pub enum DeploymentCommand { #[derive(Parser)] pub enum ProjectCommand { /// create an environment for this project on shuttle - New, + New { + #[arg(long, default_value_t = IDLE_MINUTES)] + /// How long to wait before putting the project in an idle state due to inactivity. 0 means the project will never idle + idle_minutes: u64, + }, /// list all projects belonging to the calling account List { #[arg(long)] diff --git a/cargo-shuttle/src/client.rs b/cargo-shuttle/src/client.rs index 6cab317ed..28ded40ee 100644 --- a/cargo-shuttle/src/client.rs +++ b/cargo-shuttle/src/client.rs @@ -2,11 +2,11 @@ use std::fmt::Write; use anyhow::{Context, Result}; use headers::{Authorization, HeaderMapExt}; -use reqwest::{Body, Response}; +use reqwest::Response; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware, RequestBuilder}; use reqwest_retry::policies::ExponentialBackoff; use reqwest_retry::RetryTransientMiddleware; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use shuttle_common::models::{deployment, project, secret, service, ToJson}; use shuttle_common::project::ProjectName; use shuttle_common::{ApiKey, ApiUrl, LogItem}; @@ -49,7 +49,16 @@ impl Client { let _ = write!(path, "?no-test"); } - self.post(path, Some(data)) + let url = format!("{}{}", self.api_url, path); + + let mut builder = Self::get_retry_client().post(url); + + builder = self.set_builder_auth(builder); + + builder + .body(data) + .header("Transfer-Encoding", "chunked") + .send() .await .context("failed to send deployment to the Shuttle server")? .to_json() @@ -86,10 +95,14 @@ impl Client { self.get(path).await } - pub async fn create_project(&self, project: &ProjectName) -> Result { + pub async fn create_project( + &self, + project: &ProjectName, + config: project::Config, + ) -> Result { let path = format!("/projects/{}", project.as_str()); - self.post(path, Option::::None) + self.post(path, Some(config)) .await .context("failed to make create project request")? .to_json() @@ -221,11 +234,7 @@ impl Client { .await } - async fn post>( - &self, - path: String, - body: Option, - ) -> Result { + async fn post(&self, path: String, body: Option) -> Result { let url = format!("{}{}", self.api_url, path); let mut builder = Self::get_retry_client().post(url); @@ -233,11 +242,12 @@ impl Client { builder = self.set_builder_auth(builder); if let Some(body) = body { + let body = serde_json::to_string(&body)?; builder = builder.body(body); - builder = builder.header("Transfer-Encoding", "chunked"); + builder = builder.header("Content-Type", "application/json"); } - builder.send().await + Ok(builder.send().await?) } async fn delete(&self, path: String) -> Result diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index c6d149d42..5f97aa675 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -5,7 +5,7 @@ mod factory; mod init; use indicatif::ProgressBar; -use shuttle_common::models::project::State; +use shuttle_common::models::project::{State, IDLE_MINUTES}; use shuttle_common::project::ProjectName; use std::collections::BTreeMap; @@ -99,7 +99,9 @@ impl Shuttle { Command::Stop => self.stop(&client).await, Command::Clean => self.clean(&client).await, Command::Secrets => self.secrets(&client).await, - Command::Project(ProjectCommand::New) => self.project_create(&client).await, + Command::Project(ProjectCommand::New { idle_minutes }) => { + self.project_create(&client, idle_minutes).await + } Command::Project(ProjectCommand::Status { follow }) => { self.project_status(&client, follow).await } @@ -208,7 +210,7 @@ impl Shuttle { self.load_project(&mut project_args)?; let mut client = Client::new(self.ctx.api_url()); client.set_api_key(self.ctx.api_key()?); - self.project_create(&client).await?; + self.project_create(&client, IDLE_MINUTES).await?; } Ok(()) @@ -548,7 +550,9 @@ impl Shuttle { } } - async fn project_create(&self, client: &Client) -> Result<()> { + async fn project_create(&self, client: &Client, idle_minutes: u64) -> Result<()> { + let config = project::Config { idle_minutes }; + self.wait_with_spinner( &[ project::State::Ready, @@ -556,7 +560,7 @@ impl Shuttle { message: Default::default(), }, ], - Client::create_project, + client.create_project(self.ctx.project_name(), config), self.ctx.project_name(), client, ) @@ -597,7 +601,7 @@ impl Shuttle { message: Default::default(), }, ], - Client::get_project, + client.get_project(self.ctx.project_name()), self.ctx.project_name(), client, ) @@ -612,18 +616,17 @@ impl Shuttle { Ok(()) } - async fn wait_with_spinner<'a, F, Fut>( + async fn wait_with_spinner<'a, Fut>( &self, states_to_check: &[project::State], - f: F, + fut: Fut, project_name: &'a ProjectName, client: &'a Client, ) -> Result<(), anyhow::Error> where - F: Fn(&'a Client, &'a ProjectName) -> Fut, Fut: std::future::Future> + 'a, { - let mut project = f(client, project_name).await?; + let mut project = fut.await?; let progress_bar = create_spinner(); loop { @@ -647,7 +650,7 @@ impl Shuttle { message: Default::default(), }, ], - Client::delete_project, + client.delete_project(self.ctx.project_name()), self.ctx.project_name(), client, ) diff --git a/common/Cargo.toml b/common/Cargo.toml index 0ddc8e29a..b197ddb0f 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-common" -version.workspace = true +version = "0.11.2" edition.workspace = true license.workspace = true repository.workspace = true diff --git a/common/src/backends/auth.rs b/common/src/backends/auth.rs index 959accc20..f36029211 100644 --- a/common/src/backends/auth.rs +++ b/common/src/backends/auth.rs @@ -1,11 +1,4 @@ -use std::{ - convert::Infallible, - future::Future, - ops::Add, - pin::Pin, - sync::Arc, - task::{Context, Poll}, -}; +use std::{convert::Infallible, future::Future, ops::Add, pin::Pin, sync::Arc}; use async_trait::async_trait; use bytes::Bytes; @@ -17,7 +10,6 @@ use hyper::{body, Body, Client}; use jsonwebtoken::{decode, encode, DecodingKey, EncodingKey, Header as JwtHeader, Validation}; use opentelemetry::global; use opentelemetry_http::HeaderInjector; -use pin_project::pin_project; use serde::{Deserialize, Serialize}; use thiserror::Error; use tower::{Layer, Service}; @@ -26,6 +18,7 @@ use tracing_opentelemetry::OpenTelemetrySpanExt; use super::{ cache::{CacheManagement, CacheManager}, + future::{ResponseFuture, StatusCodeFuture}, headers::XShuttleAdminSecret, }; @@ -62,17 +55,16 @@ pub struct AdminSecret { secret: String, } -impl Service> for AdminSecret +impl Service> for AdminSecret where - S: Service, Response = Response>> + S: Service, Response = Response>> + Send + 'static, S::Future: Send + 'static, { type Response = S::Response; type Error = S::Error; - type Future = - Pin> + Send + 'static>>; + type Future = StatusCodeFuture; fn poll_ready( &mut self, @@ -82,24 +74,14 @@ where } fn call(&mut self, req: Request) -> Self::Future { - let error = match req.headers().typed_try_get::() { - Ok(Some(secret)) if secret.0 == self.secret => None, - Ok(_) => Some(StatusCode::UNAUTHORIZED), - Err(_) => Some(StatusCode::BAD_REQUEST), - }; - - if let Some(status) = error { - // Could not validate claim - Box::pin(async move { - Ok(Response::builder() - .status(status) - .body(Default::default()) - .unwrap()) - }) - } else { - let future = self.inner.call(req); + match req.headers().typed_try_get::() { + Ok(Some(secret)) if secret.0 == self.secret => { + let future = self.inner.call(req); - Box::pin(async move { future.await }) + StatusCodeFuture::Poll(future) + } + Ok(_) => StatusCodeFuture::Code(StatusCode::UNAUTHORIZED), + Err(_) => StatusCodeFuture::Code(StatusCode::BAD_REQUEST), } } } @@ -474,25 +456,6 @@ pub struct ClaimService { inner: S, } -#[pin_project] -pub struct ClaimServiceFuture { - #[pin] - response_future: F, -} - -impl Future for ClaimServiceFuture -where - F: Future>, -{ - type Output = Result; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let this = self.project(); - - this.response_future.poll(cx) - } -} - impl Service>> for ClaimService where S: Service>> + Send + 'static, @@ -500,7 +463,7 @@ where { type Response = S::Response; type Error = S::Error; - type Future = ClaimServiceFuture; + type Future = ResponseFuture; fn poll_ready( &mut self, @@ -517,9 +480,9 @@ where } } - let response_future = self.inner.call(req); + let future = self.inner.call(req); - ClaimServiceFuture { response_future } + ResponseFuture(future) } } @@ -552,44 +515,6 @@ pub struct Scoped { inner: S, required: Vec, } -#[pin_project] -pub struct ScopedFuture { - #[pin] - state: ResponseState, -} - -#[pin_project(project = ResponseStateProj)] -pub enum ResponseState { - Called { - #[pin] - inner: F, - }, - Unauthorized, - Forbidden, -} - -impl Future for ScopedFuture -where - F: Future>, -{ - type Output = Result; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let this = self.project(); - - match this.state.project() { - ResponseStateProj::Called { inner } => inner.poll(cx), - ResponseStateProj::Unauthorized => Poll::Ready(Ok(Response::builder() - .status(StatusCode::UNAUTHORIZED) - .body(Default::default()) - .unwrap())), - ResponseStateProj::Forbidden => Poll::Ready(Ok(Response::builder() - .status(StatusCode::FORBIDDEN) - .body(Default::default()) - .unwrap())), - } - } -} impl Service> for Scoped where @@ -601,7 +526,7 @@ where { type Response = S::Response; type Error = S::Error; - type Future = ScopedFuture; + type Future = StatusCodeFuture; fn poll_ready( &mut self, @@ -614,7 +539,7 @@ where let Some(claim) = req.extensions().get::() else { error!("claim extension is not set"); - return ScopedFuture {state: ResponseState::Unauthorized}; + return StatusCodeFuture::Code(StatusCode::UNAUTHORIZED); }; if self @@ -623,15 +548,9 @@ where .all(|scope| claim.scopes.contains(scope)) { let response_future = self.inner.call(req); - ScopedFuture { - state: ResponseState::Called { - inner: response_future, - }, - } + StatusCodeFuture::Poll(response_future) } else { - ScopedFuture { - state: ResponseState::Forbidden, - } + StatusCodeFuture::Code(StatusCode::FORBIDDEN) } } } diff --git a/common/src/backends/future.rs b/common/src/backends/future.rs new file mode 100644 index 000000000..5603fdaa0 --- /dev/null +++ b/common/src/backends/future.rs @@ -0,0 +1,55 @@ +use std::{ + future::Future, + pin::Pin, + task::{Context, Poll}, +}; + +use axum::response::Response; +use http::StatusCode; +use pin_project::pin_project; + +// Future for layers that just return the inner response +#[pin_project] +pub struct ResponseFuture(#[pin] pub F); + +impl Future for ResponseFuture +where + F: Future>, +{ + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.project(); + + this.0.poll(cx) + } +} + +/// Future for layers that might return a different status code +#[pin_project(project = StatusCodeProj)] +pub enum StatusCodeFuture { + // A future that should be polled + Poll(#[pin] F), + + // A status code to return + Code(StatusCode), +} + +impl Future for StatusCodeFuture +where + F: Future>, +{ + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.project(); + + match this { + StatusCodeProj::Poll(inner) => inner.poll(cx), + StatusCodeProj::Code(status_code) => Poll::Ready(Ok(Response::builder() + .status(*status_code) + .body(Default::default()) + .unwrap())), + } + } +} diff --git a/common/src/backends/mod.rs b/common/src/backends/mod.rs index a72fc261e..091d671b0 100644 --- a/common/src/backends/mod.rs +++ b/common/src/backends/mod.rs @@ -1,5 +1,6 @@ pub mod auth; pub mod cache; +mod future; pub mod headers; pub mod metrics; pub mod tracing; diff --git a/common/src/backends/tracing.rs b/common/src/backends/tracing.rs index aadf74e5b..64a95cb8f 100644 --- a/common/src/backends/tracing.rs +++ b/common/src/backends/tracing.rs @@ -1,4 +1,8 @@ -use std::{future::Future, pin::Pin}; +use std::{ + future::Future, + pin::Pin, + task::{Context, Poll}, +}; use http::{Request, Response}; use opentelemetry::{ @@ -9,11 +13,14 @@ use opentelemetry::{ }; use opentelemetry_http::{HeaderExtractor, HeaderInjector}; use opentelemetry_otlp::WithExportConfig; +use pin_project::pin_project; use tower::{Layer, Service}; -use tracing::{debug_span, Span, Subscriber}; +use tracing::{debug_span, instrument::Instrumented, Instrument, Span, Subscriber}; use tracing_opentelemetry::OpenTelemetrySpanExt; use tracing_subscriber::{fmt, prelude::*, registry::LookupSpan, EnvFilter}; +use super::future::ResponseFuture; + pub fn setup_tracing(subscriber: S, service_name: &str) where S: Subscriber + for<'a> LookupSpan<'a> + Send + Sync, @@ -67,6 +74,36 @@ pub struct ExtractPropagation { inner: S, } +#[pin_project] +pub struct ExtractPropagationFuture { + #[pin] + response_future: F, +} + +impl Future for ExtractPropagationFuture +where + F: Future, Error>>, +{ + type Output = Result, Error>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.project(); + + match this.response_future.poll(cx) { + Poll::Ready(result) => match result { + Ok(response) => { + Span::current().record("http.status_code", response.status().as_u16()); + + Poll::Ready(Ok(response)) + } + other => Poll::Ready(other), + }, + + Poll::Pending => Poll::Pending, + } + } +} + impl Service> for ExtractPropagation where S: Service, Response = Response> + Send + 'static, @@ -74,8 +111,7 @@ where { type Response = S::Response; type Error = S::Error; - type Future = - Pin> + Send + 'static>>; + type Future = ExtractPropagationFuture>; fn poll_ready( &mut self, @@ -95,21 +131,12 @@ where let parent_context = global::get_text_map_propagator(|propagator| { propagator.extract(&HeaderExtractor(req.headers())) }); - span.set_parent(parent_context); - let future = self.inner.call(req); + span.set_parent(parent_context); - Box::pin(async move { - let _guard = span.enter(); + let response_future = self.inner.call(req).instrument(span); - match future.await { - Ok(response) => { - span.record("http.status_code", response.status().as_u16()); - Ok(response) - } - other => other, - } - }) + ExtractPropagationFuture { response_future } } } @@ -137,8 +164,7 @@ where { type Response = S::Response; type Error = S::Error; - type Future = - Pin> + Send + 'static>>; + type Future = ResponseFuture; fn poll_ready( &mut self, @@ -156,6 +182,6 @@ where let future = self.inner.call(req); - Box::pin(async move { future.await }) + ResponseFuture(future) } } diff --git a/common/src/models/project.rs b/common/src/models/project.rs index 14da8c924..eac73b113 100644 --- a/common/src/models/project.rs +++ b/common/src/models/project.rs @@ -7,6 +7,9 @@ use serde::{Deserialize, Serialize}; use std::fmt::{Display, Formatter}; use strum::EnumString; +// Timeframe before a project is considered idle +pub const IDLE_MINUTES: u64 = 30; + #[derive(Deserialize, Serialize, Clone)] pub struct Response { pub name: String, @@ -155,6 +158,12 @@ impl State { } } +/// Config when creating a new project +#[derive(Deserialize, Serialize)] +pub struct Config { + pub idle_minutes: u64, +} + #[derive(Deserialize, Serialize)] pub struct AdminResponse { pub project_name: String, diff --git a/deployer/Cargo.toml b/deployer/Cargo.toml index 921016895..de6dfdc74 100644 --- a/deployer/Cargo.toml +++ b/deployer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-deployer" -version.workspace = true +version = "0.11.1" edition.workspace = true license.workspace = true description = "Service with instances created per project for handling the compilation, loading, and execution of Shuttle services" diff --git a/deployer/prepare.sh b/deployer/prepare.sh index 70eac8a23..38a848028 100755 --- a/deployer/prepare.sh +++ b/deployer/prepare.sh @@ -15,12 +15,20 @@ shuttle-shared-db = { path = "/usr/src/shuttle/resources/shared-db" } shuttle-secrets = { path = "/usr/src/shuttle/resources/secrets" } shuttle-static-folder = { path = "/usr/src/shuttle/resources/static-folder" }' > $CARGO_HOME/config.toml -# Make future crates requests to our own mirror -echo ' +while getopts "p," o; do + case $o in + "p") + # Make future crates requests to our own mirror + echo ' [source.shuttle-crates-io-mirror] registry = "http://panamax:8080/git/crates.io-index" [source.crates-io] replace-with = "shuttle-crates-io-mirror"' >> $CARGO_HOME/config.toml + ;; + *) + ;; + esac +done # Prefetch crates.io index from our mirror # TODO: restore when we know how to prefetch from our mirror diff --git a/deployer/src/handlers/error.rs b/deployer/src/handlers/error.rs index f0fb98a0a..6ded00b06 100644 --- a/deployer/src/handlers/error.rs +++ b/deployer/src/handlers/error.rs @@ -6,6 +6,7 @@ use axum::Json; use serde::{ser::SerializeMap, Serialize}; use shuttle_common::models::error::ApiError; +use tracing::error; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -40,6 +41,8 @@ impl Serialize for Error { impl IntoResponse for Error { fn into_response(self) -> Response { + error!(error = &self as &dyn std::error::Error, "request error"); + let code = match self { Error::NotFound => StatusCode::NOT_FOUND, _ => StatusCode::INTERNAL_SERVER_ERROR, diff --git a/deployer/src/persistence/mod.rs b/deployer/src/persistence/mod.rs index ec6f6e6bf..91b64ef68 100644 --- a/deployer/src/persistence/mod.rs +++ b/deployer/src/persistence/mod.rs @@ -20,9 +20,7 @@ use chrono::Utc; use serde_json::json; use shuttle_common::STATE_MESSAGE; use sqlx::migrate::{MigrateDatabase, Migrator}; -use sqlx::sqlite::{ - Sqlite, SqliteConnectOptions, SqliteJournalMode, SqlitePool, SqliteSynchronous, -}; +use sqlx::sqlite::{Sqlite, SqliteConnectOptions, SqliteJournalMode, SqlitePool}; use tokio::sync::broadcast::{self, Receiver, Sender}; use tokio::task::JoinHandle; use tracing::{error, info, instrument, trace}; @@ -63,10 +61,17 @@ impl Persistence { std::fs::canonicalize(path).unwrap().to_string_lossy() ); + // We have found in the past that setting synchronous to anything other than the default (full) breaks the + // broadcast channel in deployer. The broken symptoms are that the ws socket connections won't get any logs + // from the broadcast channel and would then close. When users did deploys, this would make it seem like the + // deploy is done (while it is still building for most of the time) and the status of the previous deployment + // would be returned to the user. + // + // If you want to activate a faster synchronous mode, then also do proper testing to confirm this bug is no + // longer present. let sqlite_options = SqliteConnectOptions::from_str(path) .unwrap() - .journal_mode(SqliteJournalMode::Wal) - .synchronous(SqliteSynchronous::Normal); + .journal_mode(SqliteJournalMode::Wal); let pool = SqlitePool::connect_with(sqlite_options).await.unwrap(); diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index dcfe18a19..ea1ac7eb2 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -21,3 +21,5 @@ services: >&2 echo "DBs are available - starting provisioner" exec /usr/local/bin/service "$${@:0}" + ports: + - 5000:8000 diff --git a/docker-compose.yml b/docker-compose.yml index 85ef0e204..6fffc6dec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -173,6 +173,8 @@ services: constraints: - node.hostname==controller panamax: + profiles: + - panamax image: "${CONTAINER_REGISTRY}/panamax:${PANAMAX_TAG}" restart: always networks: @@ -189,6 +191,8 @@ services: constraints: - node.hostname==controller deck-chores: + profiles: + - panamax image: funkyfuture/deck-chores:1 restart: unless-stopped environment: diff --git a/e2e/README.md b/e2e/README.md index 7d6caaf99..2da03d7f5 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -1,9 +1,13 @@ -# Overview +# e2e + +## Overview + This crate runs all the end-to-end tests for shuttle. These tests must run against a local dev environment, so you first have to set that up by following [these instructions](../CONTRIBUTING.md). Running all the end-to-end tests may take a long time, so it is recommended to run individual tests shipped as part of each crate in the workspace first. ## Running the tests + In the root of the repository, run: ```bash diff --git a/extras/otel/otel-collector-config.yaml b/extras/otel/otel-collector-config.yaml index 7c9373356..b34f371cd 100644 --- a/extras/otel/otel-collector-config.yaml +++ b/extras/otel/otel-collector-config.yaml @@ -65,6 +65,6 @@ service: processors: [attributes, batch] exporters: [datadog] metrics: - receivers: [hostmetrics, prometheus/otel, docker_stats, otlp] + receivers: [hostmetrics, prometheus/otel, otlp] processors: [batch] exporters: [datadog] diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index bb133fe61..de6267b5a 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-gateway" -version.workspace = true +version = "0.11.2" edition.workspace = true license.workspace = true publish = false diff --git a/gateway/README.md b/gateway/README.md index 8450081ab..8797b724b 100644 --- a/gateway/README.md +++ b/gateway/README.md @@ -1,7 +1,8 @@ # shuttle-gateway ## Tests -To run the tests for gateway, follow the steps in [contributing](https://github.com/shuttle-hq/shuttle/blob/main/CONTRIBUTING.md) to set up your local environment. Then, from the root of the repository, run: + +To run the tests for gateway, follow the steps in [contributing](../CONTRIBUTING.md) to set up your local environment. Then, from the root of the repository, run: ```bash SHUTTLE_TESTS_RUNTIME_IMAGE=public.ecr.aws/shuttle-dev/deployer:latest SHUTTLE_TESTS_NETWORK=shuttle-dev_user-net cargo test --package shuttle-gateway --all-features -- --nocapture diff --git a/gateway/src/api/latest.rs b/gateway/src/api/latest.rs index 1fd1f14eb..2b64bc4fb 100644 --- a/gateway/src/api/latest.rs +++ b/gateway/src/api/latest.rs @@ -32,7 +32,7 @@ use uuid::Uuid; use crate::acme::{AcmeClient, CustomDomain}; use crate::auth::{ScopedUser, User}; -use crate::project::{Project, ProjectCreating}; +use crate::project::{ContainerInspectResponseExt, Project, ProjectCreating}; use crate::task::{self, BoxedTask, TaskResult}; use crate::tls::GatewayCertResolver; use crate::worker::WORKER_QUEUE_SIZE; @@ -131,11 +131,12 @@ async fn post_project( }): State, User { name, claim, .. }: User, Path(project): Path, + AxumJson(config): AxumJson, ) -> Result, Error> { let is_admin = claim.scopes.contains(&Scope::Admin); let state = service - .create_project(project.clone(), name.clone(), is_admin) + .create_project(project.clone(), name.clone(), is_admin, config.idle_minutes) .await?; service @@ -185,11 +186,18 @@ async fn delete_project( #[instrument(skip_all, fields(scope = %scoped_user.scope))] async fn route_project( - State(RouterState { service, .. }): State, + State(RouterState { + service, sender, .. + }): State, scoped_user: ScopedUser, req: Request, ) -> Result, Error> { - service.route(&scoped_user, req).await + let project_name = scoped_user.scope; + let project = service.find_or_start_project(&project_name, sender).await?; + + service + .route(&project, &project_name, &scoped_user.user.name, req) + .await } async fn get_status(State(RouterState { sender, .. }): State) -> Response { @@ -335,6 +343,9 @@ async fn request_acme_certificate( Err(err) => return Err(err), }; + let project = service.find_project(&project_name).await?; + let idle_minutes = project.container().unwrap().idle_minutes(); + // destroy and recreate the project with the new domain service .new_task() @@ -346,8 +357,11 @@ async fn request_acme_certificate( move |ctx| { let fqdn = fqdn.clone(); async move { - let creating = ProjectCreating::new_with_random_initial_key(ctx.project_name) - .with_fqdn(fqdn); + let creating = ProjectCreating::new_with_random_initial_key( + ctx.project_name, + idle_minutes, + ) + .with_fqdn(fqdn); TaskResult::Done(Project::Creating(creating)) } } diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index 86529e4d0..b1a6e6175 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -770,13 +770,14 @@ pub mod tests { let api_client = world.client(world.args.control); let api = ApiBuilder::new() .with_service(Arc::clone(&service)) - .with_sender(log_out) + .with_sender(log_out.clone()) .with_default_routes() .with_auth_service(world.context().auth_uri) .binding_to(world.args.control); let user = UserServiceBuilder::new() .with_service(Arc::clone(&service)) + .with_task_sender(log_out.clone()) .with_public(world.fqdn()) .with_user_proxy_binding_to(world.args.user); diff --git a/gateway/src/main.rs b/gateway/src/main.rs index c41220541..ecad9833b 100644 --- a/gateway/src/main.rs +++ b/gateway/src/main.rs @@ -93,8 +93,12 @@ async fn start(db: SqlitePool, fs: PathBuf, args: StartArgs) -> io::Result<()> { let gateway = Arc::clone(&gateway); let sender = sender.clone(); async move { + let mut interval = tokio::time::interval(Duration::from_secs(60)); + interval.tick().await; // first tick is immediate + loop { - tokio::time::sleep(Duration::from_secs(60)).await; + interval.tick().await; + if sender.capacity() < WORKER_QUEUE_SIZE - SVC_DEGRADED_THRESHOLD { // if degraded, don't stack more health checks warn!( @@ -138,11 +142,12 @@ async fn start(db: SqlitePool, fs: PathBuf, args: StartArgs) -> io::Result<()> { let mut api_builder = ApiBuilder::new() .with_service(Arc::clone(&gateway)) - .with_sender(sender) + .with_sender(sender.clone()) .binding_to(args.control); let mut user_builder = UserServiceBuilder::new() .with_service(Arc::clone(&gateway)) + .with_task_sender(sender) .with_public(args.context.proxy_fqdn.clone()) .with_user_proxy_binding_to(args.user) .with_bouncer(args.bouncer); diff --git a/gateway/src/project.rs b/gateway/src/project.rs index 23d95a881..da6dc12c7 100644 --- a/gateway/src/project.rs +++ b/gateway/src/project.rs @@ -1,10 +1,11 @@ -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use std::convert::{identity, Infallible}; use std::net::{IpAddr, SocketAddr}; use std::time::Duration; use bollard::container::{ - Config, CreateContainerOptions, RemoveContainerOptions, StopContainerOptions, + Config, CreateContainerOptions, KillContainerOptions, RemoveContainerOptions, Stats, + StatsOptions, StopContainerOptions, }; use bollard::errors::Error as DockerError; use bollard::models::{ContainerInspectResponse, ContainerStateStatusEnum}; @@ -19,6 +20,7 @@ use hyper::Client; use once_cell::sync::Lazy; use rand::distributions::{Alphanumeric, DistString}; use serde::{Deserialize, Serialize}; +use shuttle_common::models::project::IDLE_MINUTES; use tokio::time::{sleep, timeout}; use tracing::{debug, error, info, instrument}; @@ -97,6 +99,20 @@ pub trait ContainerInspectResponseExt { .map_err(|_| ProjectError::internal("invalid project name")) } + fn idle_minutes(&self) -> u64 { + let container = self.container(); + + if let Some(config) = &container.config { + if let Some(labels) = &config.labels { + if let Some(idle_minutes) = labels.get("shuttle.idle_minutes") { + return idle_minutes.parse::().unwrap_or(IDLE_MINUTES); + } + } + } + + IDLE_MINUTES + } + fn find_arg_and_then<'s, F, O>(&'s self, find: &str, and_then: F) -> Result where F: FnOnce(&'s str) -> O, @@ -204,6 +220,20 @@ impl Project { } } + pub fn start(self) -> Result { + if let Some(container) = self.container() { + Ok(Self::Starting(ProjectStarting { + container, + restart_count: 0, + })) + } else { + Err(Error::custom( + ErrorKind::InvalidOperation, + format!("cannot start a project in the `{}` state", self.state()), + )) + } + } + pub fn is_ready(&self) -> bool { matches!(self, Self::Ready(_)) } @@ -212,6 +242,10 @@ impl Project { matches!(self, Self::Destroyed(_)) } + pub fn is_stopped(&self) -> bool { + matches!(self, Self::Stopped(_)) + } + pub fn target_ip(&self) -> Result, Error> { match self.clone() { Self::Ready(project_ready) => Ok(Some(*project_ready.target_ip())), @@ -372,6 +406,7 @@ where Self::Started(started) => match started.next(ctx).await { Ok(ProjectReadying::Ready(ready)) => Ok(ready.into()), Ok(ProjectReadying::Started(started)) => Ok(started.into()), + Ok(ProjectReadying::Idle(stopping)) => Ok(stopping.into()), Err(err) => Ok(Self::Errored(err)), }, Self::Ready(ready) => ready.next(ctx).await.into_try_state(), @@ -447,7 +482,7 @@ where { Ok(container) => match safe_unwrap!(container.state.status) { ContainerStateStatusEnum::RUNNING => { - Self::Started(ProjectStarted::new(container)) + Self::Started(ProjectStarted::new(container, VecDeque::new())) } ContainerStateStatusEnum::CREATED => Self::Starting(ProjectStarting { container, @@ -470,8 +505,8 @@ where } Err(err) => return Err(err.into()), }, - Self::Started(ProjectStarted { container, .. }) - | Self::Ready(ProjectReady { container, .. }) + Self::Started(ProjectStarted { container, stats, .. }) + | Self::Ready(ProjectReady { container, stats, .. }) => match container .clone() .refresh(ctx) @@ -479,7 +514,7 @@ where { Ok(container) => match safe_unwrap!(container.state.status) { ContainerStateStatusEnum::RUNNING => { - Self::Started(ProjectStarted::new(container)) + Self::Started(ProjectStarted::new(container, stats)) } // Restart the container if it went down ContainerStateStatusEnum::EXITED => Self::Restarting(ProjectRestarting { container, restart_count: 0 }), @@ -546,10 +581,12 @@ pub struct ProjectCreating { // Use default for backward compatibility. Can be removed when all projects in the DB have this property set #[serde(default)] recreate_count: usize, + /// Label set on container as to how many minutes to wait before a project is considered idle + idle_minutes: u64, } impl ProjectCreating { - pub fn new(project_name: ProjectName, initial_key: String) -> Self { + pub fn new(project_name: ProjectName, initial_key: String, idle_minutes: u64) -> Self { Self { project_name, initial_key, @@ -557,6 +594,7 @@ impl ProjectCreating { image: None, from: None, recreate_count: 0, + idle_minutes, } } @@ -565,6 +603,7 @@ impl ProjectCreating { recreate_count: usize, ) -> Result { let project_name = container.project_name()?; + let idle_minutes = container.idle_minutes(); let initial_key = container.initial_key()?; Ok(Self { @@ -574,6 +613,7 @@ impl ProjectCreating { image: None, from: Some(container), recreate_count, + idle_minutes, }) } @@ -587,9 +627,9 @@ impl ProjectCreating { self } - pub fn new_with_random_initial_key(project_name: ProjectName) -> Self { + pub fn new_with_random_initial_key(project_name: ProjectName, idle_minutes: u64) -> Self { let initial_key = Alphanumeric.sample_string(&mut rand::thread_rng(), 32); - Self::new(project_name, initial_key) + Self::new(project_name, initial_key, idle_minutes) } pub fn with_image(mut self, image: String) -> Self { @@ -635,6 +675,7 @@ impl ProjectCreating { project_name, fqdn, image, + idle_minutes, .. } = &self; @@ -653,6 +694,7 @@ impl ProjectCreating { "Labels": { "shuttle.prefix": prefix, "shuttle.project": project_name, + "shuttle.idle_minutes": format!("{idle_minutes}"), }, "Cmd": [ "--admin-secret", @@ -720,7 +762,8 @@ where #[instrument(skip_all)] async fn next(self, ctx: &Ctx) -> Result { let container_name = self.container_name(ctx); - let recreate_count = self.recreate_count; + let Self { recreate_count, .. } = self; + let container = ctx .docker() // If container already exists, use that @@ -908,7 +951,7 @@ where let container = container.refresh(ctx).await?; - Ok(Self::Next::new(container)) + Ok(Self::Next::new(container, VecDeque::new())) } } @@ -960,13 +1003,17 @@ where pub struct ProjectStarted { container: ContainerInspectResponse, service: Option, + // Use default for backward compatibility. Can be removed when all projects in the DB have this property set + #[serde(default)] + stats: VecDeque, } impl ProjectStarted { - pub fn new(container: ContainerInspectResponse) -> Self { + pub fn new(container: ContainerInspectResponse, stats: VecDeque) -> Self { Self { container, service: None, + stats, } } } @@ -975,6 +1022,7 @@ impl ProjectStarted { pub enum ProjectReadying { Ready(ProjectReady), Started(ProjectStarted), + Idle(ProjectStopping), } #[async_trait] @@ -987,14 +1035,91 @@ where #[instrument(skip_all)] async fn next(self, ctx: &Ctx) -> Result { - let container = self.container.refresh(ctx).await?; - let mut service = match self.service { + let Self { + container, + service, + mut stats, + } = self; + let container = container.refresh(ctx).await?; + let mut service = match service { Some(service) => service, None => Service::from_container(container.clone())?, }; if service.is_healthy().await { - Ok(Self::Next::Ready(ProjectReady { container, service })) + let idle_minutes = container.idle_minutes(); + + // Idle minutes of `0` means it is disabled and the project will always stay up + if idle_minutes < 1 { + Ok(Self::Next::Ready(ProjectReady { + container, + service, + stats, + })) + } else { + let new_stat = ctx + .docker() + .stats( + safe_unwrap!(container.id), + Some(StatsOptions { + one_shot: true, + stream: false, + }), + ) + .next() + .await + .unwrap()?; + + stats.push_back(new_stat.clone()); + + let mut last = None; + + while stats.len() > (idle_minutes as usize) { + last = stats.pop_front(); + } + + if let Some(last) = last { + let cpu_per_minute = (new_stat.cpu_stats.cpu_usage.total_usage + - last.cpu_stats.cpu_usage.total_usage) + / idle_minutes; + + debug!( + "{} has {} CPU usage per minute", + service.name, cpu_per_minute + ); + + // From analysis we know the following kind of CPU usage for different kinds of idle projects + // Web framework uses 6_200_000 CPU per minute + // Serenity uses 20_000_000 CPU per minute + // + // We want to make sure we are able to stop these kinds of projects + // + // Now, the following kind of CPU usage has been observed for different kinds of projects having + // 2 web requests / processing 2 discord messages per minute + // Web framework uses 100_000_000 CPU per minute + // Serenity uses 30_000_000 CPU per minute + // + // And projects at these levels we will want to keep active. However, the 30_000_000 + // for an "active" discord will be to close to the 20_000_000 of an idle framework. And + // discord will have more traffic in anyway. So using the 100_000_000 threshold of an + // active framework for now + if cpu_per_minute < 100_000_000 { + Ok(Self::Next::Idle(ProjectStopping { container })) + } else { + Ok(Self::Next::Ready(ProjectReady { + container, + service, + stats, + })) + } + } else { + Ok(Self::Next::Ready(ProjectReady { + container, + service, + stats, + })) + } + } } else { let started_at = chrono::DateTime::parse_from_rfc3339(safe_unwrap!(container.state.started_at)) @@ -1011,6 +1136,7 @@ where Ok(Self::Next::Started(ProjectStarted { container, service: Some(service), + stats, })) } } @@ -1020,6 +1146,9 @@ where pub struct ProjectReady { container: ContainerInspectResponse, service: Service, + // Use default for backward compatibility. Can be removed when all projects in the DB have this property set + #[serde(default)] + stats: VecDeque, } #[async_trait] @@ -1188,10 +1317,18 @@ where #[instrument(skip_all)] async fn next(self, ctx: &Ctx) -> Result { let Self { container } = self; + + // Stopping a docker containers sends a SIGTERM which will stop the tokio runtime that deployer starts up. + // Killing this runtime causes the deployment to enter the `completed` state and it therefore does not + // start up again when starting up the project's container. Luckily the kill command allows us to change the + // signal to prevent this from happenning. + // + // In some future state when all deployers hadle `SIGTERM` correctly, this can be changed to docker stop + // safely. ctx.docker() - .stop_container( + .kill_container( safe_unwrap!(container.id), - Some(StopContainerOptions { t: 30 }), + Some(KillContainerOptions { signal: "SIGKILL" }), ) .await?; Ok(Self::Next { @@ -1505,6 +1642,7 @@ pub mod tests { image: None, from: None, recreate_count: 0, + idle_minutes: 0, }), #[assertion = "Container created, attach network"] Ok(Project::Attaching(ProjectAttaching { diff --git a/gateway/src/proxy.rs b/gateway/src/proxy.rs index 3e6c05936..986ed46d6 100644 --- a/gateway/src/proxy.rs +++ b/gateway/src/proxy.rs @@ -23,12 +23,14 @@ use once_cell::sync::Lazy; use opentelemetry::global; use opentelemetry_http::HeaderInjector; use shuttle_common::backends::headers::XShuttleProject; +use tokio::sync::mpsc::Sender; use tower::{Service, ServiceBuilder}; use tracing::{debug_span, error, field, trace}; use tracing_opentelemetry::OpenTelemetrySpanExt; use crate::acme::{AcmeClient, ChallengeResponderLayer, CustomDomain}; use crate::service::GatewayService; +use crate::task::BoxedTask; use crate::{Error, ErrorKind}; static PROXY_CLIENT: Lazy>> = @@ -69,6 +71,7 @@ where #[derive(Clone)] pub struct UserProxy { gateway: Arc, + task_sender: Sender, remote_addr: SocketAddr, public: FQDN, } @@ -82,7 +85,11 @@ impl<'r> AsResponderTo<&'r AddrStream> for UserProxy { } impl UserProxy { - async fn proxy(self, mut req: Request) -> Result { + async fn proxy( + self, + task_sender: Sender, + mut req: Request, + ) -> Result { let span = debug_span!("proxy", http.method = %req.method(), http.host = ?req.headers().get("Host"), http.uri = %req.uri(), http.status_code = field::Empty, project = field::Empty); trace!(?req, "serving proxy request"); @@ -111,7 +118,10 @@ impl UserProxy { req.headers_mut() .typed_insert(XShuttleProject(project_name.to_string())); - let project = self.gateway.find_project(&project_name).await?; + let project = self + .gateway + .find_or_start_project(&project_name, task_sender) + .await?; // Record current project for tracing purposes span.record("project", &project_name.to_string()); @@ -153,8 +163,9 @@ impl Service> for UserProxy { } fn call(&mut self, req: Request) -> Self::Future { + let task_sender = self.task_sender.clone(); self.clone() - .proxy(req) + .proxy(task_sender, req) .or_else(|err: Error| future::ready(Ok(err.into_response()))) .boxed() } @@ -219,6 +230,7 @@ impl Service> for Bouncer { pub struct UserServiceBuilder { service: Option>, + task_sender: Option>, acme: Option, tls_acceptor: Option>, bouncer_binds_to: Option, @@ -236,6 +248,7 @@ impl UserServiceBuilder { pub fn new() -> Self { Self { service: None, + task_sender: None, public: None, acme: None, tls_acceptor: None, @@ -254,6 +267,11 @@ impl UserServiceBuilder { self } + pub fn with_task_sender(mut self, task_sender: Sender) -> Self { + self.task_sender = Some(task_sender); + self + } + pub fn with_bouncer(mut self, bound_to: SocketAddr) -> Self { self.bouncer_binds_to = Some(bound_to); self @@ -276,6 +294,7 @@ impl UserServiceBuilder { pub fn serve(self) -> impl Future> { let service = self.service.expect("a GatewayService is required"); + let task_sender = self.task_sender.expect("a task sender is required"); let public = self.public.expect("a public FQDN is required"); let user_binds_to = self .user_binds_to @@ -283,6 +302,7 @@ impl UserServiceBuilder { let user_proxy = UserProxy { gateway: service.clone(), + task_sender, remote_addr: "127.0.0.1:80".parse().unwrap(), public: public.clone(), }; diff --git a/gateway/src/service.rs b/gateway/src/service.rs index 88ce51299..7bd55b61e 100644 --- a/gateway/src/service.rs +++ b/gateway/src/service.rs @@ -20,14 +20,14 @@ use sqlx::migrate::Migrator; use sqlx::sqlite::SqlitePool; use sqlx::types::Json as SqlxJson; use sqlx::{query, Error as SqlxError, Row}; -use tracing::{debug, Span}; +use tokio::sync::mpsc::Sender; +use tracing::{debug, trace, Span}; use tracing_opentelemetry::OpenTelemetrySpanExt; use crate::acme::CustomDomain; use crate::args::ContextArgs; -use crate::auth::ScopedUser; use crate::project::{Project, ProjectCreating}; -use crate::task::{BoxedTask, TaskBuilder}; +use crate::task::{self, BoxedTask, TaskBuilder}; use crate::worker::TaskRouter; use crate::{AccountName, DockerContext, Error, ErrorKind, ProjectDetails, ProjectName}; @@ -202,13 +202,12 @@ impl GatewayService { pub async fn route( &self, - scoped_user: &ScopedUser, + project: &Project, + project_name: &ProjectName, + account_name: &AccountName, mut req: Request, ) -> Result, Error> { - let project_name = &scoped_user.scope; - let target_ip = self - .find_project(project_name) - .await? + let target_ip = project .target_ip()? .ok_or_else(|| Error::from_kind(ErrorKind::ProjectNotReady))?; @@ -219,7 +218,7 @@ impl GatewayService { let control_key = self.control_key_from_project_name(project_name).await?; let headers = req.headers_mut(); - headers.typed_insert(XShuttleAccountName(scoped_user.user.name.to_string())); + headers.typed_insert(XShuttleAccountName(account_name.to_string())); headers.typed_insert(XShuttleAdminSecret(control_key)); let cx = Span::current().context(); @@ -362,6 +361,7 @@ impl GatewayService { project_name: ProjectName, account_name: AccountName, is_admin: bool, + idle_minutes: u64, ) -> Result { if let Some(row) = query( r#" @@ -381,8 +381,10 @@ impl GatewayService { let project = row.get::, _>("project_state").0; if project.is_destroyed() { // But is in `::Destroyed` state, recreate it - let mut creating = - ProjectCreating::new_with_random_initial_key(project_name.clone()); + let mut creating = ProjectCreating::new_with_random_initial_key( + project_name.clone(), + idle_minutes, + ); // Restore previous custom domain, if any match self.find_custom_domain_for_project(&project_name).await { Ok(custom_domain) => { @@ -409,7 +411,8 @@ impl GatewayService { // Otherwise attempt to create a new one. This will fail // outright if the project already exists (this happens if // it belongs to another account). - self.insert_project(project_name, account_name).await + self.insert_project(project_name, account_name, idle_minutes) + .await } else { Err(Error::from_kind(ErrorKind::InvalidProjectName)) } @@ -420,9 +423,10 @@ impl GatewayService { &self, project_name: ProjectName, account_name: AccountName, + idle_minutes: u64, ) -> Result { let project = SqlxJson(Project::Creating( - ProjectCreating::new_with_random_initial_key(project_name.clone()), + ProjectCreating::new_with_random_initial_key(project_name.clone(), idle_minutes), )); query("INSERT INTO projects (project_name, account_name, initial_key, project_state) VALUES (?1, ?2, ?3, ?4)") @@ -545,6 +549,35 @@ impl GatewayService { TaskBuilder::new(self.clone()) } + /// Find a project by name. And start the project if it is idle, waiting for it to start up + pub async fn find_or_start_project( + self: &Arc, + project_name: &ProjectName, + task_sender: Sender, + ) -> Result { + let mut project = self.find_project(project_name).await?; + + // Start the project if it is idle + if project.is_stopped() { + trace!(%project_name, "starting up idle project"); + + let handle = self + .new_task() + .project(project_name.clone()) + .and_then(task::start()) + .and_then(task::run_until_done()) + .and_then(task::check_health()) + .send(&task_sender) + .await?; + + // Wait for project to come up and set new state + handle.await; + project = self.find_project(project_name).await?; + } + + Ok(project) + } + pub fn task_router(&self) -> TaskRouter { self.task_router.clone() } @@ -592,7 +625,7 @@ pub mod tests { }; let project = svc - .create_project(matrix.clone(), neo.clone(), false) + .create_project(matrix.clone(), neo.clone(), false, 0) .await .unwrap(); @@ -653,7 +686,7 @@ pub mod tests { // If recreated by a different user assert!(matches!( - svc.create_project(matrix.clone(), trinity.clone(), false) + svc.create_project(matrix.clone(), trinity.clone(), false, 0) .await, Err(Error { kind: ErrorKind::ProjectAlreadyExists, @@ -663,7 +696,7 @@ pub mod tests { // If recreated by the same user assert!(matches!( - svc.create_project(matrix.clone(), neo, false).await, + svc.create_project(matrix.clone(), neo, false, 0).await, Ok(Project::Creating(_)) )); @@ -684,7 +717,7 @@ pub mod tests { // If recreated by an admin assert!(matches!( - svc.create_project(matrix, trinity, true).await, + svc.create_project(matrix, trinity, true, 0).await, Ok(Project::Creating(_)) )); @@ -699,7 +732,7 @@ pub mod tests { let neo: AccountName = "neo".parse().unwrap(); let matrix: ProjectName = "matrix".parse().unwrap(); - svc.create_project(matrix.clone(), neo.clone(), false) + svc.create_project(matrix.clone(), neo.clone(), false, 0) .await .unwrap(); @@ -764,7 +797,7 @@ pub mod tests { ); let _ = svc - .create_project(project_name.clone(), account.clone(), false) + .create_project(project_name.clone(), account.clone(), false, 0) .await .unwrap(); @@ -818,7 +851,7 @@ pub mod tests { ); let _ = svc - .create_project(project_name.clone(), account.clone(), false) + .create_project(project_name.clone(), account.clone(), false, 0) .await .unwrap(); @@ -836,7 +869,7 @@ pub mod tests { assert!(matches!(work.poll(()).await, TaskResult::Done(()))); let recreated_project = svc - .create_project(project_name.clone(), account.clone(), false) + .create_project(project_name.clone(), account.clone(), false, 0) .await .unwrap(); diff --git a/gateway/src/task.rs b/gateway/src/task.rs index 499417260..fabd2292d 100644 --- a/gateway/src/task.rs +++ b/gateway/src/task.rs @@ -126,6 +126,15 @@ pub fn destroy() -> impl Task { }) } +pub fn start() -> impl Task { + run(|ctx| async move { + match ctx.state.start() { + Ok(state) => TaskResult::Done(state), + Err(err) => TaskResult::Err(err), + } + }) +} + pub fn check_health() -> impl Task { run(|ctx| async move { match ctx.state.refresh(&ctx.gateway).await { diff --git a/provisioner/Cargo.toml b/provisioner/Cargo.toml index 4a6caaea8..794a19f02 100644 --- a/provisioner/Cargo.toml +++ b/provisioner/Cargo.toml @@ -12,7 +12,7 @@ aws-config = "0.51.0" aws-sdk-rds = "0.21.0" clap = { workspace = true, features = ["env"] } fqdn = "0.2.3" -mongodb = "2.3.1" +mongodb = "2.4.0" prost = "0.11.2" rand = { workspace = true } sqlx = { version = "0.6.2", features = [ diff --git a/resources/README.md b/resources/README.md index d6e00cd77..531322c0a 100644 --- a/resources/README.md +++ b/resources/README.md @@ -1,5 +1,11 @@ +# Resources + ## Managed resources -The list of managed resources for shuttle is always growing. If you feel we are missing a resource you would like, then feel to create a feature request for your desired resource. + +The list of managed resources for shuttle is always growing. +If you feel we are missing a resource you would like, then feel to create a feature request for your desired resource. ## Writing your own managed resources -Creating your own resources is actually easy. You only need to implement the [`ResourceBuilder`](https://docs.rs/shuttle-service/latest/shuttle_service/trait.ResourceBuilder.html) trait for your resource. + +Creating your own resources is actually easy. +You only need to implement the [`ResourceBuilder`](https://docs.rs/shuttle-service/latest/shuttle_service/trait.ResourceBuilder.html) trait for your resource. diff --git a/resources/aws-rds/README.md b/resources/aws-rds/README.md index d3f311575..a8bf7099f 100644 --- a/resources/aws-rds/README.md +++ b/resources/aws-rds/README.md @@ -1,10 +1,13 @@ # Shuttle AWS RDS + This plugin provisions databases on AWS RDS using [shuttle](https://www.shuttle.rs). The following three engines are supported: + - Postgres - MySql - MariaDB ## Usage + Add `shuttle-aws-rds` to the dependencies for your service. Every engine is behind the following feature flags and attribute paths: | Engine | Feature flag | Attribute path | @@ -16,6 +19,7 @@ Add `shuttle-aws-rds` to the dependencies for your service. Every engine is behi An example using the Tide framework can be found on [GitHub](https://github.com/shuttle-hq/examples/tree/main/tide/postgres) ### Options + Each engine can take in the following options: | Option | Type | Description | diff --git a/resources/persist/README.md b/resources/persist/README.md index 333bf75ba..c829ee10b 100644 --- a/resources/persist/README.md +++ b/resources/persist/README.md @@ -1,8 +1,9 @@ # Shuttle Persist + This plugin allows persisting struct that implement `serde::Serialize` and loading them again using `serde::Deserialize`. ## Usage + Add `shuttle-persist` to the dependencies for your service. You can get this resource using the `shuttle-persist::Persist` attribute to get a `PersistInstance`. Object can now be saved using `PersistInstance.save()` and loaded again using `PersistInstance.load()`. An example using the Rocket framework can be found on [GitHub](https://github.com/shuttle-hq/examples/tree/main/rocket/persist) - diff --git a/resources/secrets/README.md b/resources/secrets/README.md index 08e7912a5..1d2a5ca84 100644 --- a/resources/secrets/README.md +++ b/resources/secrets/README.md @@ -1,7 +1,9 @@ # Shuttle Secrets + This plugin manages secrets on [shuttle](https://www.shuttle.rs). ## Usage + Add `shuttle-secrets` to the dependencies for your service, and add a `Secrets.toml` to the root of your project with the secrets you'd like to store. Make sure to add `Secrets.toml` to a `.gitignore` to omit your secrets from version control. diff --git a/resources/shared-db/README.md b/resources/shared-db/README.md index 68a36e7b2..fad4bdfa5 100644 --- a/resources/shared-db/README.md +++ b/resources/shared-db/README.md @@ -1,7 +1,9 @@ # Shuttle Shared Databases + This plugin manages databases that are shared with other services on [shuttle](https://www.shuttle.rs). ## Usage + Add `shuttle-shared-db` to the dependencies for your service. Every type of shareable database is behind the following feature flag and attribute path | Engine | Feature flag | Attribute path | @@ -12,6 +14,7 @@ Add `shuttle-shared-db` to the dependencies for your service. Every type of shar An example using the Rocket framework can be found on [GitHub](https://github.com/shuttle-hq/examples/tree/main/rocket/postgres) ### Postgres + This resource has the following options | Option | Type | Description | @@ -19,9 +22,9 @@ This resource has the following options | local_uri | &str | Don't spin a local docker instance of Postgres, but rather connect to this URI instead for `cargo shuttle run` | ### MongoDB + This resource has the following options | Option | Type | Description | |-----------|------|---------------------------------------------------------------------------------------------------------------| | local_uri | &str | Don't spin a local docker instance of MongoDB, but rather connect to this URI instead for `cargo shuttle run` | - diff --git a/resources/static-folder/README.md b/resources/static-folder/README.md index 48e27ba32..272381ed0 100644 --- a/resources/static-folder/README.md +++ b/resources/static-folder/README.md @@ -1,7 +1,9 @@ # Shuttle Static Folder + This plugin allows services to get the path to a static folder at runtime ## Usage + Add `shuttle-static-folder` to the dependencies for your service. This resource can be using by the `shuttle_static_folder::StaticFolder` attribute to get a `PathBuf` with the location of the static folder. An example using the Axum framework can be found on [GitHub](https://github.com/shuttle-hq/examples/tree/main/axum/websocket) @@ -14,11 +16,13 @@ async fn main( ``` ### Parameters + | Parameter | Type | Default | Description | |-----------|------|----------|--------------------------------------------------------------------| | folder | str | `static` | The relative path, from the crate root, to the directory containing static files to deploy | ### Example: Using the public folder instead + Since this plugin defaults to the `static` folder, the arguments can be used to use the `public` folder instead. ``` rust diff --git a/service/Cargo.toml b/service/Cargo.toml index 2513eddcb..8fc56086e 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -27,11 +27,11 @@ libloading = { version = "0.7.4", optional = true } num_cpus = { version = "1.14.0", optional = true } pipe = "0.4.0" poem = { version = "1.3.49", optional = true } +poise = { version = "0.5.2", optional = true } rocket = { version = "0.5.0-rc.2", optional = true } salvo = { version = "0.37.5", optional = true } serde_json = { workspace = true } serenity = { version = "0.11.5", default-features = false, features = ["client", "gateway", "rustls_backend", "model"], optional = true } -poise = { version = "0.5.2", optional = true } strfmt = "0.2.2" sync_wrapper = { version = "0.1.1", optional = true } thiserror = { workspace = true } @@ -71,13 +71,13 @@ loader = ["cargo", "libloading"] web-actix-web = ["actix-web", "num_cpus"] web-axum = ["axum", "sync_wrapper"] +web-poem = ["poem"] web-rocket = ["rocket"] +web-salvo = ["salvo"] web-thruster = ["thruster"] web-tide = ["tide", "async-std"] web-tower = ["tower", "hyper"] -web-poem = ["poem"] -web-salvo = ["salvo"] +web-warp = ["warp"] -bot-serenity = ["serenity"] bot-poise = ["poise"] -web-warp = ["warp"] +bot-serenity = ["serenity"]