From 5efb7e2c61d26749eea7b319f83ec932b391dda4 Mon Sep 17 00:00:00 2001 From: Lucas Pickering Date: Mon, 3 Jun 2024 17:08:17 -0400 Subject: [PATCH] Add !json variant for structured bodies - Add !json tag to `body` field of recipes - !json bodies can contain any data. Strings will be treated as templates The original design in the ticket was to use have the user still provide `Content-Type`, but I realized there isn't much value in that. It just requires more work for the user, and introduces another thing we need to validate (body type vs `Content-Type`). If the user needs a subset of `application/json` they can still override it. Otherwise, providing it ourselves is just more convenient. The other motivating factor behind is forms. The machinery that reqwest gives us to set form data automatically sets the header, so I think it makes sense to follow that lead. Closes #242 --- CHANGELOG.md | 5 +- Cargo.lock | 12 + Cargo.toml | 3 +- docs/src/SUMMARY.md | 10 +- .../api/request_collection/authentication.md | 14 +- docs/src/api/request_collection/chain.md | 14 +- .../api/request_collection/chain_source.md | 12 +- .../api/request_collection/content_type.md | 11 - docs/src/api/request_collection/index.md | 12 +- .../src/api/request_collection/recipe_body.md | 49 +++ .../api/request_collection/request_recipe.md | 12 +- docs/src/getting_started.md | 3 +- docs/src/user_guide/chains.md | 14 +- docs/src/user_guide/filter_query.md | 9 +- docs/src/user_guide/inheritance.md | 5 +- docs/src/user_guide/templates.md | 3 +- slumber.yml | 16 +- src/collection.rs | 307 +++++++++++++++- src/collection/cereal.rs | 342 ++++++++++++++---- src/collection/insomnia.rs | 52 ++- src/collection/models.rs | 142 +++++++- src/http.rs | 341 ++++++++++++----- src/http/content_type.rs | 143 ++++---- src/http/models.rs | 7 + src/template.rs | 2 +- src/tui/view/common/text_window.rs | 2 + src/tui/view/component/recipe_pane.rs | 106 ++++-- test_data/insomnia.json | 151 +++++--- test_data/insomnia_imported.yml | 69 ++-- test_data/regression.yml | 131 +++++++ 30 files changed, 1555 insertions(+), 444 deletions(-) delete mode 100644 docs/src/api/request_collection/content_type.md create mode 100644 docs/src/api/request_collection/recipe_body.md create mode 100644 test_data/regression.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b0c2d45..e83962da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Added +- JSON bodies can now be defined with the `!json` tag [#242](https://github.com/LucasPickering/slumber/issues/242) + - This should make JSON requests more convenient to write, because you no longer have to specify the `Content-Type` header yourself + - [See docs](https://slumber.lucaspickering.me/book/api/request_collection/recipe_body.html) - Templates can now render binary values in certain contexts - - [See docs](https://slumber.lucaspickering.me/book/user_guide/templates.html#binary-templates) for more info + - [See docs](https://slumber.lucaspickering.me/book/user_guide/templates.html#binary-templates) ### Changed diff --git a/Cargo.lock b/Cargo.lock index ff7fb577..cbd68fec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -127,6 +127,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.65", +] + [[package]] name = "async-trait" version = "0.1.80" @@ -2000,6 +2011,7 @@ name = "slumber" version = "1.3.2" dependencies = [ "anyhow", + "async-recursion", "async-trait", "bytes", "bytesize", diff --git a/Cargo.toml b/Cargo.toml index 8af7bd61..c09585e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ rust-version = "1.76.0" [dependencies] anyhow = {version = "^1.0.75", features = ["backtrace"]} +async-recursion = "1.1.1" async-trait = "^0.1.73" bytes = {version = "1.5.0", features = ["serde"]} bytesize = {version = "1.3.0", default-features = false} @@ -31,7 +32,6 @@ mime = "^0.3.17" nom = "7.1.3" notify = {version = "^6.1.1", default-features = false, features = ["macos_fsevent"]} open = "5.1.1" -pretty_assertions = "1.4.0" ratatui = {version = "^0.26.0", features = ["serde", "unstable-rendered-line-info"]} reqwest = {version = "^0.12.4", default-features = false, features = ["rustls-tls"]} rmp-serde = "^1.1.2" @@ -51,6 +51,7 @@ uuid = {version = "^1.4.1", default-features = false, features = ["serde", "v4"] [dev-dependencies] mockito = {version = "1.4.0", default-features = false} +pretty_assertions = "1.4.0" rstest = {version = "0.19.0", default-features = false} serde_test = "1.0.176" diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index faf4bf36..c200eb1c 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -28,11 +28,11 @@ - [Request Collection](./api/request_collection/index.md) - [Profile](./api/request_collection/profile.md) - [Request Recipe](./api/request_collection/request_recipe.md) - - [Authentication](./api/request_collection/authentication.md) - - [Chain](./api/request_collection/chain.md) - - [Chain Source](./api/request_collection/chain_source.md) - - [Template](./api/request_collection/template.md) - - [Content Type](./api/request_collection/content_type.md) + - [Authentication](./api/request_collection/authentication.md) + - [Recipe Body](./api/request_collection/recipe_body.md) + - [Chain](./api/request_collection/chain.md) + - [Chain Source](./api/request_collection/chain_source.md) + - [Template](./api/request_collection/template.md) - [Configuration](./api/configuration/index.md) - [Input Bindings](./api/configuration/input_bindings.md) - [Theme](./api/configuration/theme.md) diff --git a/docs/src/api/request_collection/authentication.md b/docs/src/api/request_collection/authentication.md index 99d9117f..c0f7f392 100644 --- a/docs/src/api/request_collection/authentication.md +++ b/docs/src/api/request_collection/authentication.md @@ -4,10 +4,10 @@ Authentication provides shortcuts for common HTTP authentication schemes. It pop ## Variants -| Variant | Type | Value | -| -------- | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | -| `basic` | [`Basic Authentication`](#basic-authentication) | [Basic authentication](https://swagger.io/docs/specification/authentication/basic-authentication/) credentials | -| `bearer` | `string` | [Bearer token](https://swagger.io/docs/specification/authentication/bearer-authentication/) | +| Variant | Type | Value | +| --------- | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| `!basic` | [`Basic Authentication`](#basic-authentication) | [Basic authentication](https://swagger.io/docs/specification/authentication/basic-authentication/) credentials | +| `!bearer` | `string` | [Bearer token](https://swagger.io/docs/specification/authentication/bearer-authentication/) | ### Basic Authentication @@ -26,8 +26,7 @@ requests: create_fish: !request method: POST url: "{{host}}/fishes" - body: > - {"kind": "barracuda", "name": "Jimmy"} + body: !json { "kind": "barracuda", "name": "Jimmy" } authentication: !basic username: user password: pass @@ -41,7 +40,6 @@ requests: create_fish: !request method: POST url: "{{host}}/fishes" - body: > - {"kind": "barracuda", "name": "Jimmy"} + body: !json { "kind": "barracuda", "name": "Jimmy" } authentication: !bearer "{{chains.token}}" ``` diff --git a/docs/src/api/request_collection/chain.md b/docs/src/api/request_collection/chain.md index f8abab2b..112d15a2 100644 --- a/docs/src/api/request_collection/chain.md +++ b/docs/src/api/request_collection/chain.md @@ -6,13 +6,13 @@ To use a chain in a template, reference it as `{{chains.}}`. ## Fields -| Field | Type | Description | Default | -| -------------- | -------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | -------- | -| `source` | [`ChainSource`](./chain_source.md) | Source of the chained value | Required | -| `sensitive` | `boolean` | Should the value be hidden in the UI? | `false` | -| `selector` | [`JSONPath`](https://www.ietf.org/archive/id/draft-goessner-dispatch-jsonpath-00.html) | Selector to transform/narrow down results in a chained value. See [Filtering & Querying](../../user_guide/filter_query.md) | `null` | -| `content_type` | [`ContentType`](./content_type.md) | Force content type. Not required for `request` and `file` chains, as long as the `Content-Type` header/file extension matches the data | | -| `trim` | [`ChainOutputTrim`](#chain-output-trim) | Trim whitespace from the rendered output | `none` | +| Field | Type | Description | Default | +| -------------- | -------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| `source` | [`ChainSource`](./chain_source.md) | Source of the chained value | Required | +| `sensitive` | `boolean` | Should the value be hidden in the UI? | `false` | +| `selector` | [`JSONPath`](https://www.ietf.org/archive/id/draft-goessner-dispatch-jsonpath-00.html) | Selector to transform/narrow down results in a chained value. See [Filtering & Querying](../../user_guide/filter_query.md) | `null` | +| `content_type` | `string` | Force content type. Not required for `request` and `file` chains, as long as the `Content-Type` header/file extension matches the data. See [here](./recipe_body.md#supported-content-types) for a list of supported types. | | +| `trim` | [`ChainOutputTrim`](#chain-output-trim) | Trim whitespace from the rendered output | `none` | See the [`ChainSource`](./chain_source.md) docs for detail on the different types of chainable values. diff --git a/docs/src/api/request_collection/chain_source.md b/docs/src/api/request_collection/chain_source.md index 899ea2a5..e5f993d7 100644 --- a/docs/src/api/request_collection/chain_source.md +++ b/docs/src/api/request_collection/chain_source.md @@ -23,12 +23,12 @@ message: Enter Password ## Variants -| Variant | Type | Description | -| --------- | ---------------------------------- | --------------------------------------------------------------- | -| `request` | [`ChainSource::Request`](#request) | Body of the most recent response for a specific request recipe. | -| `command` | [`ChainSource::Command`](#command) | Stdout of the executed command | -| `file` | [`ChainSource::File`](#file) | Contents of the file | -| `prompt` | [`ChainSource::Prompt`](#prompt) | Value entered by the user | +| Variant | Type | Description | +| ---------- | ---------------------------------- | --------------------------------------------------------------- | +| `!request` | [`ChainSource::Request`](#request) | Body of the most recent response for a specific request recipe. | +| `!command` | [`ChainSource::Command`](#command) | Stdout of the executed command | +| `!file` | [`ChainSource::File`](#file) | Contents of the file | +| `!prompt` | [`ChainSource::Prompt`](#prompt) | Value entered by the user | ### Request diff --git a/docs/src/api/request_collection/content_type.md b/docs/src/api/request_collection/content_type.md deleted file mode 100644 index 2561194d..00000000 --- a/docs/src/api/request_collection/content_type.md +++ /dev/null @@ -1,11 +0,0 @@ -# Content Type - -Content type defines the various data formats that Slumber recognizes and can manipulate. Slumber is capable of displaying any text-based data format, but only specific formats support additional features such as [querying](../../user_guide/filter_query.md) and formatting. - -For chained requests, Slumber uses the [HTTP `Content-Type` header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type) to detect the content type. For chained files, it uses the file extension. For other [chain sources](./chain_source.md), or if the `Content-Type` header/file extension is missing or incorrect, you'll have to manually provide the content type via the [chain](./chain.md) `content_type` field. - -## Supported Content Types - -| Content Type | HTTP Header | File Extension(s) | -| ------------ | ------------------ | ----------------- | -| JSON | `application/json` | `json` | diff --git a/docs/src/api/request_collection/index.md b/docs/src/api/request_collection/index.md index 65534a8d..d5c0b843 100644 --- a/docs/src/api/request_collection/index.md +++ b/docs/src/api/request_collection/index.md @@ -58,22 +58,21 @@ chains: recipe: login selector: $.token -# Use YAML anchors for de-duplication (Anything under .ignore is ignored) +# Use YAML anchors for de-duplication (Anything under .ignore will not trigger an error for unknown fields) .ignore: base: &base headers: Accept: application/json - Content-Type: application/json requests: login: !request <<: *base method: POST url: "{{host}}/anything/login" - body: | - { + body: + !json { "username": "{{chains.username}}", - "password": "{{chains.password}}" + "password": "{{chains.password}}", } # Folders can be used to keep your recipes organized @@ -92,6 +91,5 @@ requests: method: PUT url: "{{host}}/anything/current-user" authentication: !bearer "{{chains.auth_token}}" - body: > - {"username": "Kenny"} + body: !json { "username": "Kenny" } ``` diff --git a/docs/src/api/request_collection/recipe_body.md b/docs/src/api/request_collection/recipe_body.md new file mode 100644 index 00000000..118add25 --- /dev/null +++ b/docs/src/api/request_collection/recipe_body.md @@ -0,0 +1,49 @@ +# Recipe Body + +There are a variety of ways to define the body of your request. Slumber supports structured bodies for a fixed set of known content types (see table below). + +In addition, you can pass any [`Template`](./template.md) to render any text or binary data. In this case, you'll probably want to explicitly set the `Content-Type` header to tell the server what kind of data you're sending. This may not be necessary though, depending on the server implementation. + +## Supported Content Types + +The following content types have first-class support. The meaning of each column is as follows: + +- Variant - Tag used when defining a body of this type +- Type - Type of the associated value +- `Content-Type` - Value to insert for the `Content-Type` request header + - Additionally, this is the associated content type when determining how to [parse chained responses for the purposes of filtering/querying](../../user_guide/filter_query.md). +- File Extension - File extension(s) that map to this content type when determining how to [parse chained files for the purposes of filtering/querying](../../user_guide/filter_query.md) + +| Variant | Type | `Content-Type` | File Extension | Description | +| ------- | ---- | ------------------ | -------------- | ---------------------------------------------------------------- | +| `!json` | Any | `application/json` | `json` | Structured JSON body, where all strings are treated as templates | + +## Examples + +```yaml +chains: + image: + source: !file + path: ./fish.png + +requests: + text_body: !request + method: POST + url: "{{host}}/fishes/{{fish_id}}/name" + headers: + Content-Type: text/plain + body: Balthazar + + binary_body: !request + method: POST + url: "{{host}}/fishes/{{fish_id}}/image" + headers: + Content-Type: image/jpg + body: "{{chains.fish_image}}" + + json_body: !request + method: POST + url: "{{host}}/fishes/{{fish_id}}" + # Content-Type header will be set automatically based on the body type + body: !json { "name": "Balthazar" } +``` diff --git a/docs/src/api/request_collection/request_recipe.md b/docs/src/api/request_collection/request_recipe.md index e1b8c934..e193e45f 100644 --- a/docs/src/api/request_collection/request_recipe.md +++ b/docs/src/api/request_collection/request_recipe.md @@ -18,7 +18,7 @@ The tag for a recipe is `!request` (see examples). | `query` | [`mapping[string, Template]`](./template.md) | HTTP request query parameters | `{}` | | `headers` | [`mapping[string, Template]`](./template.md) | HTTP request headers | `{}` | | `authentication` | [`Authentication`](./authentication.md) | Authentication scheme | `null` | -| `body` | [`Template`](./template.md) | HTTP request body | `null` | +| `body` | [`RecipeBody`](./recipe_body.md) | HTTP request body | `null` | ## Folder Fields @@ -39,13 +39,12 @@ recipes: url: "{{host}}/anything/login" headers: accept: application/json - content-type: application/json query: root_access: yes_please - body: | - { + body: + !json { "username": "{{chains.username}}", - "password": "{{chains.password}}" + "password": "{{chains.password}}", } fish: !folder name: Users @@ -53,8 +52,7 @@ recipes: create_fish: !request method: POST url: "{{host}}/fishes" - body: > - {"kind": "barracuda", "name": "Jimmy"} + body: !json { "kind": "barracuda", "name": "Jimmy" } list_fish: !request method: GET diff --git a/docs/src/getting_started.md b/docs/src/getting_started.md index c3ca16fa..e8e0c330 100644 --- a/docs/src/getting_started.md +++ b/docs/src/getting_started.md @@ -43,8 +43,7 @@ requests: create_fish: !request method: POST url: "{{host}}/fishes" - body: > - {"kind": "barracuda", "name": "Jimmy"} + body: !json { "kind": "barracuda", "name": "Jimmy" } list_fish: !request method: GET diff --git a/docs/src/user_guide/chains.md b/docs/src/user_guide/chains.md index 2806996f..b10c8d84 100644 --- a/docs/src/user_guide/chains.md +++ b/docs/src/user_guide/chains.md @@ -24,10 +24,10 @@ requests: login: !request method: POST url: "https://myfishes.fish/login" - body: | - { + body: + !json { "username": "{{chains.username}}", - "password": "{{chains.password}}" + "password": "{{chains.password}}", } get_user: !request @@ -99,17 +99,17 @@ chains: recipe: login auth_token: source: !command - command: [ "cut", "-d':'", "-f2" ] + command: ["cut", "-d':'", "-f2"] stdin: "{{chains.auth_token_raw}}" requests: login: !request method: POST url: "https://myfishes.fish/login" - body: | - { + body: + !json { "username": "{{chains.username}}", - "password": "{{chains.password}}" + "password": "{{chains.password}}", } get_user: !request diff --git a/docs/src/user_guide/filter_query.md b/docs/src/user_guide/filter_query.md index b823e225..89997bb8 100644 --- a/docs/src/user_guide/filter_query.md +++ b/docs/src/user_guide/filter_query.md @@ -27,6 +27,7 @@ We'll use these credentials to log in and get an API token, so the second data s ```yaml chains: username: + # Slumber knows how to query this file based on its extension source: !file path: ./creds.json selector: $.user @@ -43,10 +44,10 @@ requests: login: !request method: POST url: "https://myfishes.fish/anything/login" - body: | - { + body: + !json { "username": "{{chains.username}}", - "password": "{{chains.password}}" + "password": "{{chains.password}}", } get_user: !request @@ -84,7 +85,7 @@ requests: login: !request method: POST url: "https://myfishes.fish/anything/login" - body: | + body: !json { "username": "{{chains.username}}", "password": "{{chains.password}}" diff --git a/docs/src/user_guide/inheritance.md b/docs/src/user_guide/inheritance.md index 2ab9e1ee..c57a28d3 100644 --- a/docs/src/user_guide/inheritance.md +++ b/docs/src/user_guide/inheritance.md @@ -115,7 +115,6 @@ requests: url: "{{host}}/fishes" headers: <<: *headers_base - Content-Type: application/json - body: > - {"kind": "barracuda", "name": "Jimmy"} + Host: myfishes.fish + body: !json { "kind": "barracuda", "name": "Jimmy" } ``` diff --git a/docs/src/user_guide/templates.md b/docs/src/user_guide/templates.md index 9300d6e7..4e0c4210 100644 --- a/docs/src/user_guide/templates.md +++ b/docs/src/user_guide/templates.md @@ -74,8 +74,7 @@ requests: create_fish: !request method: POST url: "{{host}}/fishes" - body: > - {"kind": "barracuda", "name": "Jimmy"} + body: !json { "kind": "barracuda", "name": "Jimmy" } get_fish: !request method: GET diff --git a/slumber.yml b/slumber.yml index 5b192294..2718091c 100644 --- a/slumber.yml +++ b/slumber.yml @@ -28,14 +28,13 @@ chains: source: !request recipe: login trigger: !expire 12h - selector: $.headers["X-Amzn-Trace-Id"] + selector: $.data .ignore: base: &base authentication: !bearer "{{chains.auth_token}}" headers: Accept: application/json - Content-Type: application/json requests: login: !request @@ -46,12 +45,8 @@ requests: fast: no_thanks headers: Accept: application/json - Content-Type: application/json - body: | - { - "username": "{{username}}", - "password": "{{chains.password}}" - } + body: + !json { "username": "{{username}}", "password": "{{chains.password}}" } users: !folder name: Users @@ -75,10 +70,7 @@ requests: name: Modify User method: PUT url: "{{host}}/anything/{{user_guid}}" - body: | - { - "username": "new username" - } + body: !json { "username": "new username" } get_image: !request headers: diff --git a/src/collection.rs b/src/collection.rs index c45b939c..8649696a 100644 --- a/src/collection.rs +++ b/src/collection.rs @@ -180,9 +180,16 @@ async fn load_collection(path: PathBuf) -> anyhow::Result { #[cfg(test)] mod tests { use super::*; - use crate::test_util::{assert_err, temp_dir, TempDir}; + use crate::{ + http::ContentType, + test_util::{assert_err, by_id, temp_dir, test_data_dir, TempDir}, + }; + use indexmap::indexmap; + use pretty_assertions::assert_eq; use rstest::rstest; - use std::fs::File; + use serde::de::IgnoredAny; + use serde_json::json; + use std::{fs::File, time::Duration}; /// Test various cases of try_path #[rstest] @@ -228,4 +235,300 @@ mod tests { ); drop(temp_dir); // Dropping deletes the directory } + + /// A catch-all regression test, to make sure we don't break anything in the + /// collection format. This lives at the bottom because it's huge. + #[rstest] + #[tokio::test] + async fn test_regression(test_data_dir: PathBuf) { + let loaded = CollectionFile::load(test_data_dir.join("regression.yml")) + .await + .unwrap() + .collection; + let expected = Collection { + profiles: by_id([ + Profile { + id: "profile1".into(), + name: Some("Profile 1".into()), + data: indexmap! { + "user_guid".into() => "abc123".into(), + "username".into() => "xX{{chains.username}}Xx".into(), + "host".into() => "https://httpbin.org".into(), + + }, + }, + Profile { + id: "profile2".into(), + name: Some("Profile 2".into()), + data: indexmap! { + "host".into() => "https://httpbin.org".into(), + + }, + }, + ]), + chains: by_id([ + Chain { + id: "command".into(), + source: ChainSource::command(["whoami"]), + sensitive: false, + selector: None, + content_type: None, + trim: ChainOutputTrim::None, + }, + Chain { + id: "command_stdin".into(), + source: ChainSource::Command { + command: vec!["head -c 1".into()], + stdin: Some("abcdef".into()), + }, + sensitive: false, + selector: None, + content_type: None, + trim: ChainOutputTrim::None, + }, + Chain { + id: "command_trim_none".into(), + source: ChainSource::command(["whoami"]), + sensitive: false, + selector: None, + content_type: None, + trim: ChainOutputTrim::None, + }, + Chain { + id: "command_trim_start".into(), + source: ChainSource::command(["whoami"]), + sensitive: false, + selector: None, + content_type: None, + trim: ChainOutputTrim::Start, + }, + Chain { + id: "command_trim_end".into(), + source: ChainSource::command(["whoami"]), + sensitive: false, + selector: None, + content_type: None, + trim: ChainOutputTrim::End, + }, + Chain { + id: "command_trim_both".into(), + source: ChainSource::command(["whoami"]), + sensitive: false, + selector: None, + content_type: None, + trim: ChainOutputTrim::Both, + }, + Chain { + id: "prompt_sensitive".into(), + source: ChainSource::Prompt { + message: Some("Password".into()), + default: None, + }, + sensitive: true, + selector: None, + content_type: None, + trim: ChainOutputTrim::None, + }, + Chain { + id: "prompt_default".into(), + source: ChainSource::Prompt { + message: Some("User GUID".into()), + default: Some("{{user_guid}}".into()), + }, + sensitive: false, + selector: None, + content_type: None, + trim: ChainOutputTrim::None, + }, + Chain { + id: "file".into(), + source: ChainSource::File { + path: "./README.md".into(), + }, + sensitive: false, + selector: None, + content_type: None, + trim: ChainOutputTrim::None, + }, + Chain { + id: "file_content_type".into(), + source: ChainSource::File { + path: "./data.json".into(), + }, + sensitive: false, + selector: None, + content_type: Some(ContentType::Json), + trim: ChainOutputTrim::None, + }, + Chain { + id: "request_selector".into(), + source: ChainSource::Request { + recipe: "login".into(), + trigger: ChainRequestTrigger::Never, + section: ChainRequestSection::Body, + }, + sensitive: false, + selector: Some("$.data".parse().unwrap()), + content_type: None, + trim: ChainOutputTrim::None, + }, + Chain { + id: "request_trigger_never".into(), + source: ChainSource::Request { + recipe: "login".into(), + trigger: ChainRequestTrigger::Never, + section: ChainRequestSection::Body, + }, + sensitive: false, + selector: None, + content_type: None, + trim: ChainOutputTrim::None, + }, + Chain { + id: "request_trigger_no_history".into(), + source: ChainSource::Request { + recipe: "login".into(), + trigger: ChainRequestTrigger::Never, + section: ChainRequestSection::Body, + }, + sensitive: false, + selector: None, + content_type: None, + trim: ChainOutputTrim::None, + }, + Chain { + id: "request_trigger_expire".into(), + source: ChainSource::Request { + recipe: "login".into(), + trigger: ChainRequestTrigger::Expire( + Duration::from_secs(12 * 60 * 60), + ), + section: ChainRequestSection::Body, + }, + sensitive: false, + selector: None, + content_type: None, + trim: ChainOutputTrim::None, + }, + Chain { + id: "request_trigger_always".into(), + source: ChainSource::Request { + recipe: "login".into(), + trigger: ChainRequestTrigger::Never, + section: ChainRequestSection::Body, + }, + sensitive: false, + selector: None, + content_type: None, + trim: ChainOutputTrim::None, + }, + Chain { + id: "request_section_body".into(), + source: ChainSource::Request { + recipe: "login".into(), + trigger: ChainRequestTrigger::Never, + section: ChainRequestSection::Body, + }, + sensitive: false, + selector: None, + content_type: None, + trim: ChainOutputTrim::None, + }, + Chain { + id: "request_section_header".into(), + source: ChainSource::Request { + recipe: "login".into(), + trigger: ChainRequestTrigger::Never, + section: ChainRequestSection::Header( + "content-type".into(), + ), + }, + sensitive: false, + selector: None, + content_type: None, + trim: ChainOutputTrim::None, + }, + ]), + recipes: by_id([ + RecipeNode::Recipe(Recipe { + id: "text_body".into(), + name: None, + method: Method::Post, + url: "{{host}}/anything/login".into(), + + body: Some(RecipeBody::Raw( + "{\"username\": \"{{username}}\", \ + \"password\": \"{{chains.password}}\"}" + .into(), + )), + authentication: None, + query: indexmap! { + "sudo".into() => "yes_please".into(), + "fast".into() => "no_thanks".into(), + }, + headers: indexmap! { + "Accept".into() => "application/json".into(), + }, + }), + RecipeNode::Folder(Folder { + id: "users".into(), + name: Some("Users".into()), + children: by_id([ + RecipeNode::Recipe(Recipe { + id: "simple".into(), + name: Some("Get User".into()), + method: Method::Get, + url: "{{host}}/anything/{{user_guid}}".into(), + + body: None, + authentication: None, + query: indexmap! {}, + headers: indexmap! {}, + }), + RecipeNode::Recipe(Recipe { + id: "json_body".into(), + name: Some("Modify User".into()), + method: Method::Put, + url: "{{host}}/anything/{{user_guid}}".into(), + + body: Some(RecipeBody::Json( + json!({ + "username": "new username" + }) + .into(), + )), + authentication: Some(Authentication::Bearer( + "{{chains.auth_token}}".into(), + )), + query: indexmap! {}, + headers: indexmap! { + "Accept".into() => "application/json".into(), + }, + }), + RecipeNode::Recipe(Recipe { + id: "json_body_but_not".into(), + name: Some("Modify User".into()), + method: Method::Put, + url: "{{host}}/anything/{{user_guid}}".into(), + + body: Some(RecipeBody::Json( + json!(r#"{"warning": "NOT an object"}"#).into(), + )), + authentication: Some(Authentication::Basic { + username: "{{username}}".into(), + password: Some("{{password}}".into()), + }), + query: indexmap! {}, + headers: indexmap! { + "Accept".into() => "application/json".into(), + }, + }), + ]), + }), + ]) + .into(), + _ignore: IgnoredAny, + }; + assert_eq!(loaded, expected); + } } diff --git a/src/collection/cereal.rs b/src/collection/cereal.rs index dbb2ae4a..d5498a6a 100644 --- a/src/collection/cereal.rs +++ b/src/collection/cereal.rs @@ -2,16 +2,16 @@ use crate::{ collection::{ - recipe_tree::RecipeNode, Chain, ChainId, Profile, ProfileId, Recipe, - RecipeId, + recipe_tree::RecipeNode, Chain, ChainId, JsonBody, Profile, ProfileId, + Recipe, RecipeBody, RecipeId, }, template::Template, }; use serde::{ - de::{Error, Visitor}, - Deserialize, Deserializer, + de::{EnumAccess, Error, VariantAccess, Visitor}, + Deserialize, Deserializer, Serialize, Serializer, }; -use std::hash::Hash; +use std::{fmt::Display, hash::Hash, str::FromStr}; /// A type that has an `id` field. This is ripe for a derive macro, maybe a fun /// project some day? @@ -95,6 +95,17 @@ where Ok(map) } +/// Deserialize a value using its `FromStr` implementation +pub fn deserialize_from_str<'de, D, T>(deserializer: D) -> Result +where + D: Deserializer<'de>, + T: FromStr, + T::Err: Display, +{ + let s = String::deserialize(deserializer)?; + s.parse().map_err(D::Error::custom) +} + // Custom deserializer for `Template`. This is useful for deserializing values // that are not strings, but should be treated as strings such as numbers, // booleans, and nulls. @@ -111,7 +122,7 @@ impl<'de> Deserialize<'de> for Template { where E: Error, { - Template::try_from(v.to_string()).map_err(E::custom) + self.visit_string(v.to_string()) } }; } @@ -131,12 +142,114 @@ impl<'de> Deserialize<'de> for Template { visit_primitive!(visit_i64, i64); visit_primitive!(visit_f64, f64); visit_primitive!(visit_str, &str); + + fn visit_string(self, v: String) -> Result + where + E: Error, + { + Template::try_from(v).map_err(E::custom) + } } deserializer.deserialize_any(TemplateVisitor) } } +impl RecipeBody { + // Constants for serialize/deserialization. Typically these are generated + // by macros, but we need custom implementation + const STRUCT_NAME: &'static str = "RecipeBody"; + const VARIANT_JSON: &'static str = "json"; + const ALL_VARIANTS: &'static [&'static str] = &[Self::VARIANT_JSON]; +} + +/// Custom serialization for RecipeBody, so the `Raw` variant serializes as a +/// scalar without a tag +impl Serialize for RecipeBody { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + // This involves a lot of duplication, but any abstraction will probably + // just make it worse + match self { + RecipeBody::Raw(template) => template.serialize(serializer), + RecipeBody::Json(value) => serializer.serialize_newtype_variant( + Self::STRUCT_NAME, + 1, + Self::VARIANT_JSON, + value, + ), + } + } +} + +// Custom deserialization for RecipeBody, to support raw template or structured +// body with a tag +impl<'de> Deserialize<'de> for RecipeBody { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct RecipeBodyVisitor; + + /// For all primitives, parse it as a template and create a raw body + macro_rules! visit_primitive { + ($func:ident, $type:ty) => { + fn $func(self, v: $type) -> Result + where + E: Error, + { + let template = + Template::try_from(v.to_string()).map_err(E::custom)?; + Ok(RecipeBody::Raw(template)) + } + }; + } + + impl<'de> Visitor<'de> for RecipeBodyVisitor { + type Value = RecipeBody; + + fn expecting( + &self, + formatter: &mut std::fmt::Formatter, + ) -> std::fmt::Result { + // "!" is a little wonky, but tags aren't a common YAML + // syntax so we should provide a hint to the user about what it + // means. Once they provide a tag they'll get a different error + // message if it's an unsupported tag + formatter.write_str("string, boolean, number, or tag !") + } + + visit_primitive!(visit_bool, bool); + visit_primitive!(visit_u64, u64); + visit_primitive!(visit_u128, u128); + visit_primitive!(visit_i64, i64); + visit_primitive!(visit_i128, i128); + visit_primitive!(visit_f64, f64); + visit_primitive!(visit_str, &str); + + fn visit_enum(self, data: A) -> Result + where + A: EnumAccess<'de>, + { + let (tag, value) = data.variant::()?; + match tag.as_str() { + RecipeBody::VARIANT_JSON => Ok(RecipeBody::Json( + value.newtype_variant::()?, + )), + other => Err(A::Error::unknown_variant( + other, + RecipeBody::ALL_VARIANTS, + )), + } + } + } + + deserializer.deserialize_any(RecipeBodyVisitor) + } +} + /// Serialize/deserialize a duration with unit shorthand. This does *not* handle /// subsecond precision. Supported units are: /// - s @@ -221,84 +334,23 @@ pub mod serde_duration { }; Ok(Duration::from_secs(seconds)) } - - #[cfg(test)] - mod tests { - use super::*; - use rstest::rstest; - use serde::Serialize; - use serde_test::{ - assert_de_tokens, assert_de_tokens_error, assert_ser_tokens, Token, - }; - - /// A wrapper that forces serde_test to use our custom - /// serialize/deserialize functions - #[derive(Debug, PartialEq, Serialize, Deserialize)] - #[serde(transparent)] - struct Wrap(#[serde(with = "super")] Duration); - - #[rstest] - #[case::seconds_short(Duration::from_secs(3), "3s")] - #[case::seconds_long(Duration::from_secs(3000), "3000s")] - // Subsecond precision is lost - #[case::seconds_subsecond_lost(Duration::from_millis(400), "0s")] - #[case::seconds_subsecond_round_down(Duration::from_millis(1999), "1s")] - fn test_serialize( - #[case] duration: Duration, - #[case] expected: &'static str, - ) { - assert_ser_tokens(&Wrap(duration), &[Token::String(expected)]); - } - - #[rstest] - #[case::seconds_zero("0s", Duration::from_secs(0))] - #[case::seconds_short("1s", Duration::from_secs(1))] - #[case::seconds_longer("100s", Duration::from_secs(100))] - #[case::minutes("3m", Duration::from_secs(180))] - #[case::hours("3h", Duration::from_secs(10800))] - #[case::days("2d", Duration::from_secs(172800))] - fn test_deserialize( - #[case] s: &'static str, - #[case] expected: Duration, - ) { - assert_de_tokens(&Wrap(expected), &[Token::Str(s)]) - } - - #[rstest] - #[case::negative( - "-1s", - "Invalid duration, must be `` (e.g. `12d`)" - )] - #[case::whitespace( - " 1s ", - "Invalid duration, must be `` (e.g. `12d`)" - )] - #[case::trailing_whitespace( - "1s ", - "Invalid duration, must be `` (e.g. `12d`)" - )] - #[case::decimal( - "3.5s", - "Invalid duration, must be `` (e.g. `12d`)" - )] - #[case::invalid_unit( - "3hr", - "Unknown duration unit `hr`; must be one of `s`, `m`, `h`, `d`" - )] - fn test_deserialize_error( - #[case] s: &'static str, - #[case] error: &str, - ) { - assert_de_tokens_error::(&[Token::Str(s)], error) - } - } } #[cfg(test)] mod tests { - use crate::template::Template; + use super::*; + use crate::test_util::assert_err; use rstest::rstest; - use serde_test::{assert_de_tokens, Token}; + use serde::Serialize; + use serde_json::json; + use serde_test::{ + assert_de_tokens, assert_de_tokens_error, assert_ser_tokens, Token, + }; + use serde_yaml::{ + value::{Tag, TaggedValue}, + Mapping, + }; + use std::time::Duration; #[rstest] // boolean @@ -317,4 +369,136 @@ mod tests { fn test_deserialize_template(#[case] token: Token, #[case] expected: &str) { assert_de_tokens(&Template::from(expected), &[token]); } + + /// Test serializing and deserializing recipe bodies. Round trips should all + /// be no-ops. We use serde_yaml instead of serde_test because the handling + /// of enums is a bit different, and we specifically only care about YAML. + #[rstest] + #[case::raw( + RecipeBody::Raw("{{user_id}}".into()), + "{{user_id}}" + )] + #[case::json( + RecipeBody::Json(json!({"user": "{{user_id}}"}).into()), + serde_yaml::Value::Tagged(Box::new(TaggedValue{ + tag: Tag::new("json"), + value: [ + (serde_yaml::Value::from("user"), "{{user_id}}".into()) + ].into_iter().collect::().into() + })), + )] + #[case::json_nested( + RecipeBody::Json(json!(r#"{"warning": "NOT an object"}"#).into()), + serde_yaml::Value::Tagged(Box::new(TaggedValue{ + tag: Tag::new("json"), + value: r#"{"warning": "NOT an object"}"#.into() + })), + )] + fn test_serde_recipe_body( + #[case] body: RecipeBody, + #[case] yaml: impl Into, + ) { + let yaml = yaml.into(); + assert_eq!( + serde_yaml::to_value(&body).unwrap(), + yaml, + "Serialization mismatch" + ); + assert_eq!( + serde_yaml::from_value::(yaml).unwrap(), + body, + "Deserialization mismatch" + ); + } + + /// Test various errors when deserializing a recipe body. We use serde_yaml + /// instead of serde_test because the handling of enums is a bit different, + /// and we specifically only care about YAML. + #[rstest] + #[case::array( + Vec::::new(), + "invalid type: sequence, expected string, boolean, number, or tag !" + )] + #[case::map( + Mapping::default(), + "invalid type: map, expected string, boolean, number, or tag !" + )] + // `Raw` variant is *not* accessible by tag + #[case::raw_tag( + serde_yaml::Value::Tagged(Box::new(TaggedValue{ + tag: Tag::new("raw"), + value: "{{user_id}}".into() + })), + "unknown variant `raw`, expected `json`", + )] + fn test_deserialize_recipe_error( + #[case] yaml: impl Into, + #[case] expected_error: &str, + ) { + assert_err!( + serde_yaml::from_value::(yaml.into()), + expected_error + ); + } + + /// A wrapper that forces serde_test to use our custom serialize/deserialize + /// functions + #[derive(Debug, PartialEq, Serialize, Deserialize)] + #[serde(transparent)] + struct WrapDuration(#[serde(with = "super::serde_duration")] Duration); + + #[rstest] + #[case::seconds_short(Duration::from_secs(3), "3s")] + #[case::seconds_long(Duration::from_secs(3000), "3000s")] + // Subsecond precision is lost + #[case::seconds_subsecond_lost(Duration::from_millis(400), "0s")] + #[case::seconds_subsecond_round_down(Duration::from_millis(1999), "1s")] + fn test_serialize_duration( + #[case] duration: Duration, + #[case] expected: &'static str, + ) { + assert_ser_tokens(&WrapDuration(duration), &[Token::String(expected)]); + } + + #[rstest] + #[case::seconds_zero("0s", Duration::from_secs(0))] + #[case::seconds_short("1s", Duration::from_secs(1))] + #[case::seconds_longer("100s", Duration::from_secs(100))] + #[case::minutes("3m", Duration::from_secs(180))] + #[case::hours("3h", Duration::from_secs(10800))] + #[case::days("2d", Duration::from_secs(172800))] + fn test_deserialize_duration( + #[case] s: &'static str, + #[case] expected: Duration, + ) { + assert_de_tokens(&WrapDuration(expected), &[Token::Str(s)]) + } + + #[rstest] + #[case::negative( + "-1s", + "Invalid duration, must be `` (e.g. `12d`)" + )] + #[case::whitespace( + " 1s ", + "Invalid duration, must be `` (e.g. `12d`)" + )] + #[case::trailing_whitespace( + "1s ", + "Invalid duration, must be `` (e.g. `12d`)" + )] + #[case::decimal( + "3.5s", + "Invalid duration, must be `` (e.g. `12d`)" + )] + #[case::invalid_unit( + "3hr", + "Unknown duration unit `hr`; must be one of `s`, `m`, `h`, `d`" + )] + fn test_deserialize_duration_error( + #[case] s: &'static str, + #[case] error: &str, + ) { + assert_de_tokens_error::(&[Token::Str(s)], error) + } } diff --git a/src/collection/insomnia.rs b/src/collection/insomnia.rs index 8eca6d01..f9e190c0 100644 --- a/src/collection/insomnia.rs +++ b/src/collection/insomnia.rs @@ -3,14 +3,16 @@ use crate::{ collection::{ - self, Collection, Folder, Method, Profile, ProfileId, Recipe, RecipeId, - RecipeNode, RecipeTree, + self, cereal::deserialize_from_str, Collection, Folder, JsonBody, + Method, Profile, ProfileId, Recipe, RecipeBody, RecipeId, RecipeNode, + RecipeTree, }, template::Template, }; use anyhow::{anyhow, Context}; use indexmap::IndexMap; use itertools::Itertools; +use mime::Mime; use reqwest::header; use serde::{Deserialize, Deserializer}; use std::{collections::HashMap, fs::File, path::Path}; @@ -87,6 +89,7 @@ struct Grouped { #[derive(Debug, Deserialize)] #[serde(tag = "_type", rename_all = "snake_case")] +#[allow(clippy::large_enum_variant)] enum Resource { /// Maps to a folder RequestGroup(RequestGroup), @@ -181,8 +184,11 @@ struct Parameter { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct Body { - mime_type: String, - text: Template, + #[serde(deserialize_with = "deserialize_from_str")] + mime_type: Mime, + /// This field is only present for text-like bodies (e.g. *not* forms) + #[serde(default)] + text: Option