Update CI to use cargo make (#136)

This commit simplifies the CI test flow
by

- Using rust latest docker image as base image
- Using cargo make to run the testing process

pull the docker image before building to ensure that latest tag 
is the latest release.

It appears that the -Z unstable-options --format json works 
on rust stable channel, despite rust documentation to the contrary.

It also replaces the use of ES_TEST_SERVER with
ELASTICSEARCH_URL, for consistency

.ci/DockerFile

@@ -1,57 +1,26 @@
-# rust nightly docker image is currently broken for this project.
-# Use a known, good nightly version. Since there are no docker images for specific nightly versions, build from
-# scratch, using the same commands as the rust docker image with
-# - specific nightly version that works (see RUST_TOOLCHAIN)
-# - Added openssl, libssl-dev and pkg-config packages to compile native-tls
-# FROM rustlang/rust:${RUST_TOOLCHAIN}
-FROM debian:stretch-slim
+# Use rust latest stable along with
+# - openssl, libssl-dev and pkg-config packages to compile native-tls
+# - cargo make and cargo2junit
+ARG RUST_TOOLCHAIN=latest
 
-ARG RUST_TOOLCHAIN=nightly-2020-07-27
-
-ENV RUSTUP_HOME=/usr/local/rustup \
-    CARGO_HOME=/usr/local/cargo \
-    PATH=/usr/local/cargo/bin:$PATH \
-    RUST_TOOLCHAIN=$RUST_TOOLCHAIN
+FROM rust:${RUST_TOOLCHAIN}
 
 RUN set -eux; \
     apt-get update; \
     apt-get install -y --no-install-recommends \
-        ca-certificates \
-        gcc \
-        libc6-dev \
-        wget \
         openssl \
         libssl-dev \
         pkg-config \
         ; \
-    \
-    url="https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init"; \
-    wget "$url"; \
-    chmod +x rustup-init; \
-    ./rustup-init -y --no-modify-path --default-toolchain nightly; \
-    rm rustup-init; \
-    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
-    rustup toolchain install $RUST_TOOLCHAIN; \
-    rustup default $RUST_TOOLCHAIN; \
-    rustup --version; \
-    cargo --version; \
-    rustc --version; \
-    cargo install cargo2junit; \
-    \
-    apt-get remove -y --auto-remove \
-        wget \
-        ; \
-    rm -rf /var/lib/apt/lists/*;
-
-# required to workaround a current issue with rustfmt-nightly
-ENV CFG_RELEASE=nightly
-ENV CFG_RELEASE_CHANNEL=nightly
+    cargo install --force cargo-make; \
+    cargo install cargo2junit;
 
 # create app directory
 WORKDIR /usr/src/elasticsearch-rs
 
 COPY .ci/certs ./.ci/certs
 COPY Cargo.toml ./Cargo.toml
+COPY Makefile.toml ./Makefile.toml
 COPY README.md ./README.md
 COPY api_generator ./api_generator
 COPY elasticsearch/Cargo.toml ./elasticsearch/Cargo.toml

.ci/run-repository.sh

@@ -22,22 +22,22 @@ echo -e "\033[34;1mINFO:\033[0m RUST_TOOLCHAIN ${RUST_TOOLCHAIN}\033[0m"
 
 echo -e "\033[1m>>>>> Build [elastic/elasticsearch-rs container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 
+docker pull rust:"${RUST_TOOLCHAIN}"
+
 docker build --build-arg RUST_TOOLCHAIN="${RUST_TOOLCHAIN}" --file .ci/DockerFile --tag elastic/elasticsearch-rs .
 
 echo -e "\033[1m>>>>> Run [elastic/elasticsearch-rs container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 
 repo=$(realpath $(dirname $(realpath -s $0))/../)
 
-# ES_TEST_SERVER env var is needed for cargo test
 docker run \
   --network=${network_name} \
-  --env "ES_TEST_SERVER=${ELASTICSEARCH_URL}" \
+  --env "TEST_SUITE=${TEST_SUITE}" \
+  --env "STACK_VERSION=${STACK_VERSION}" \
+  --env "ELASTICSEARCH_URL=${ELASTICSEARCH_URL}" \
+  --env "CI=true" \
   --name test-runner \
   --volume ${repo}/test_results:/usr/src/elasticsearch-rs/test_results \
   --rm \
   elastic/elasticsearch-rs \
-  /bin/bash -c \
-  "cargo run -p yaml_test_runner -- -u \"${ELASTICSEARCH_URL}\"; \\
-   mkdir -p test_results; \\
-   cargo test -p yaml_test_runner -- --test-threads=1 -Z unstable-options --format json | tee test_results/results.json; \\
-   cat test_results/results.json | cargo2junit > test_results/cargo-junit.xml"
+  /bin/bash -c "cargo make test-yaml"

.ci/test-matrix.yml

@@ -8,6 +8,6 @@ TEST_SUITE:
   - xpack
 
 RUST_TOOLCHAIN:
-  - nightly-2020-07-27
+  - latest
 
 exclude: ~

CONTRIBUTING.md

@@ -40,10 +40,11 @@ The project makes use of the following, which should be installed
 
 ### Cargo make
 
-Cargo make is used to define and configure a set of tasks, and run them as a flow. To see all of the tasks defined
+Cargo make is used to define and configure a set of tasks, and run them as a flow. To see all of the Elasticsearch
+category tasks defined
 
 ```sh
-cargo make --list-all-steps
+cargo make
 ```
 
 The `Elasticsearch` category of steps are specifically defined for this project and are defined in
@@ -69,7 +70,7 @@ The `Elasticsearch` category of steps are specifically defined for this project
   a snapshot release like `7.x-SNAPSHOT`
 
   ```sh
-  cargo make test --env STACK_VERSION=<e.g. 7.9.0>
+  cargo make test --env STACK_VERSION=7.9.0
   ```
 
 - Run YAML tests
@@ -81,7 +82,7 @@ The `Elasticsearch` category of steps are specifically defined for this project
   - `TEST_SUITE`: Elasticsearch distribution of `oss` or `xpack`
 
   ```sh
-  cargo make test-yaml --env STACK_VERSION=<e.g. 7.9.0> --env TEST_SUITE=<xpack or oss>
+  cargo make test-yaml --env STACK_VERSION=7.9.0 --env TEST_SUITE=oss
   ```
 
 ### Packages
@@ -110,7 +111,9 @@ can be `to_string()`'ed and written to disk, and this is used to create much of
   A small executable that downloads YAML tests from GitHub and generates client tests from the YAML tests. The
   version of YAML tests to download are determined from the commit hash of a running Elasticsearch instance.
 
-  The `yaml_test_runner` package can be run with `cargo test` to run the generated client tests.
+  The `yaml_test_runner` package can be run with `cargo make test-yaml` to run the generated client tests,
+  passing environment variables `TEST_SUITE` and `STACK_VERSION` to control the distribution and version,
+  respectively.
 
 ### Design principles
 
@@ -141,21 +144,13 @@ The `quote` and `syn` crates help
       An id must always be provided for a delete script API call, so the `delete_script()` function 
       must accept it as a value.
 
-### Current development setup
-
-The required toolchain for packages in the workspace are controlled
-by a `rust-toolchain` file in the root of each package.
-
-`elasticsearch` package compiles and runs with rust stable. 
-`api_generator` and `yaml_test_runner` packages require rust nightly.
-
 ### Coding style guide
 
 The repository adheres to the styling enforced by `rustfmt`.
 
 #### Formatting
 
-Rust code can be formatted using [`rustfmt`](https://github.com/rust-lang/rustfmt). Follow the instructions to install.
+Rust code can be formatted using [`rustfmt`](https://github.com/rust-lang/rustfmt) through cargo make.
 
 To format all packages in a workspace, from the workspace root
 
@@ -167,7 +162,7 @@ It is strongly recommended to run this before opening a PR.
 
 #### Clippy
 
-[Clippy](https://github.com/rust-lang/rust-clippy) is a bunch of lints to catch common mistakes and improve your Rust code! Follow the instructions to install.
+[Clippy](https://github.com/rust-lang/rust-clippy) is a bunch of lints to catch common mistakes and improve your Rust code!
 
 Run clippy before opening a PR
 

Makefile.toml

@@ -9,22 +9,22 @@ TEST_SUITE = "xpack"
 
 [tasks.set-oss-env]
 category = "Elasticsearch"
-description = "Sets ELASTICSEARCH_URL environment variable for later tasks when oss test suite used"
+description = "Sets ELASTICSEARCH_URL environment variable if not already set for later tasks when oss test suite used"
 private = true
-condition = { env = { "TEST_SUITE" = "oss" } }
+condition = { env = { "TEST_SUITE" = "oss" }, env_not_set = ["ELASTICSEARCH_URL"] }
 env = { "ELASTICSEARCH_URL" = "http://localhost:9200" }
 
 [tasks.set-xpack-env]
 category = "Elasticsearch"
-description = "Sets ELASTICSEARCH_URL environment variable for later tasks when xpack test suite used"
+description = "Sets ELASTICSEARCH_URL environment variable if not already set for later tasks when xpack test suite used"
 private = true
-condition = { env = { "TEST_SUITE" = "xpack" } }
+condition = { env = { "TEST_SUITE" = "xpack" }, env_not_set = ["ELASTICSEARCH_URL"] }
 env = { "ELASTICSEARCH_URL" = "https://elastic:changeme@localhost:9200" }
 
 [tasks.run-yaml-test-runner]
 category = "Elasticsearch"
 description = '''
-Runs yaml_test_runner crate to generate tests from yaml files for a given Elasticsearch commit.
+Runs yaml_test_runner package to generate tests from yaml files for a given Elasticsearch commit.
 The commit to use is retrieved from the running Elasticsearch instance
 '''
 private = true
@@ -37,17 +37,22 @@ script = ["cargo run -p yaml_test_runner -- -u %ELASTICSEARCH_URL%"]
 [tasks.test-yaml-test-runner]
 category = "Elasticsearch"
 private = true
-condition = { env_set = [ "ELASTICSEARCH_URL" ] }
-env = { "ES_TEST_SERVER" = "${ELASTICSEARCH_URL}" }
+condition = { env_set = [ "ELASTICSEARCH_URL" ], env_not_set = ["CI"] }
 command = "cargo"
 args = ["test", "-p", "yaml_test_runner", "--", "--test-threads=1"]
 dependencies = ["generate-yaml-tests"]
 
+[tasks.test-yaml-test-runner-ci]
+category = "Elasticsearch"
+private = true
+condition = { env_set = [ "ELASTICSEARCH_URL", "CI" ] }
+script = ["cargo test -p yaml_test_runner -- --test-threads=1 -Z unstable-options --format json | tee test_results/results.json"]
+dependencies = ["generate-yaml-tests"]
+
 [tasks.test-elasticsearch]
 category = "Elasticsearch"
 private = true
 condition = { env_set = [ "ELASTICSEARCH_URL" ], env = { "TEST_SUITE" = "xpack" } }
-env = { "ES_TEST_SERVER" = "${ELASTICSEARCH_URL}" }
 command = "cargo"
 args = ["test", "-p", "elasticsearch"]
 dependencies = ["start-elasticsearch"]
@@ -58,14 +63,32 @@ private = true
 command = "cargo"
 args = ["run", "-p", "api_generator"]
 
+[tasks.create-test-results-dir]
+category = "Elasticsearch"
+private = true
+condition = { env_set = [ "CI" ] }
+script = ["[ -d test_results ] || mkdir -p test_results"]
+
+[tasks.install-cargo2junit]
+category = "Elasticsearch"
+private = true
+script = ["cargo install cargo2junit"]
+
+[tasks.convert-test-results-junit]
+category = "Elasticsearch"
+private = true
+condition = { env_set = [ "CI" ] }
+script = ["cat test_results/results.json | cargo2junit > test_results/cargo-junit.xml"]
+dependencies = ["install-cargo2junit"]
+
 # ============
 # Public tasks
 # ============
 
 [tasks.start-elasticsearch]
 category = "Elasticsearch"
 description = "Starts Elasticsearch docker container with the given version and distribution"
-condition = { env_set = [ "STACK_VERSION", "TEST_SUITE" ] }
+condition = { env_set = [ "STACK_VERSION", "TEST_SUITE" ], env_not_set = ["CI"] }
 script = ["DETACH=true bash .ci/run-elasticsearch.sh"]
 dependencies = ["set-oss-env", "set-xpack-env"]
 
@@ -75,7 +98,7 @@ script = ["bash -c \"STACK_VERSION=%STACK_VERSION% TEST_SUITE=%TEST_SUITE% DETAC
 [tasks.stop-elasticsearch]
 category = "Elasticsearch"
 description = "Stops Elasticsearch docker container, if running"
-condition = { env_set = [ "STACK_VERSION", "TEST_SUITE" ] }
+condition = { env_set = [ "STACK_VERSION", "TEST_SUITE" ], env_not_set = ["CI"] }
 script = ["CLEANUP=true bash .ci/run-elasticsearch.sh"]
 dependencies = ["set-oss-env", "set-xpack-env"]
 
@@ -84,9 +107,9 @@ script = ["bash -c \"STACK_VERSION=%STACK_VERSION% TEST_SUITE=%TEST_SUITE% CLEAN
 
 [tasks.test-yaml]
 category = "Elasticsearch"
-description = "Generates and runs yaml_test_runner crate xpack/oss tests against a given Elasticsearch version"
+description = "Generates and runs yaml_test_runner package xpack/oss tests against a given Elasticsearch version"
 condition = { env_set = [ "STACK_VERSION", "TEST_SUITE" ] }
-dependencies = ["generate-yaml-tests", "test-yaml-test-runner"]
+dependencies = ["generate-yaml-tests", "create-test-results-dir", "test-yaml-test-runner", "test-yaml-test-runner-ci", "convert-test-results-junit"]
 run_task = "stop-elasticsearch"
 
 [tasks.test-generator]
@@ -99,7 +122,7 @@ args = ["test", "-p", "api_generator"]
 [tasks.test]
 category = "Elasticsearch"
 clear = true
-description = "Runs Elasticsearch crate tests against a given Elasticsearch version"
+description = "Runs elasticsearch package tests against a given Elasticsearch version"
 env = { "TEST_SUITE" = { value = "xpack", condition = { env_set = ["TEST_SUITE"] } } }
 dependencies = ["test-elasticsearch"]
 run_task = "stop-elasticsearch"
@@ -125,12 +148,14 @@ script = ['''
     echo "- generate-api: Generates Elasticsearch client from REST API specs"
     echo "- start-elasticsearch: Starts Elasticsearch docker container with the given version and distribution"
     echo "- stop-elasticsearch: Stops Elasticsearch docker container, if running"
-    echo "- test-yaml: Generates and runs yaml_test_runner crate xpack/oss tests against a given Elasticsearch version"
-    echo "- test: Runs Elasticsearch crate tests against a given Elasticsearch version"
+    echo "- test-yaml: Generates and runs yaml_test_runner package xpack/oss tests against a given Elasticsearch version"
+    echo "- test-generator: Generates and runs api_generator package tests"
+    echo "- test: Runs elasticsearch package tests against a given Elasticsearch version"
     echo
     echo "Most tasks use these environment variables:"
     echo "- STACK_VERSION (default '$STACK_VERSION'): the version of Elasticsearch"
     echo "- TEST_SUITE ('oss' or 'xpack', default '$TEST_SUITE'): the distribution of Elasticsearch"
+    echo "- CI (default not set): set when running on CI to determine whether to start Elasticsearch and format test output as JSON"
     echo
     echo "Run 'cargo make --list-all-steps' for a complete list of available tasks."
     echo
@@ -144,12 +169,14 @@ script = ['''
     echo - generate-api: Generates Elasticsearch client from REST API specs
     echo - start-elasticsearch: Starts Elasticsearch docker container with the given version and distribution
     echo - stop-elasticsearch: Stops Elasticsearch docker container, if running
-    echo - test-yaml: Generates and runs yaml_test_runner crate xpack/oss tests against a given Elasticsearch version
-    echo - test: Runs Elasticsearch crate tests against a given Elasticsearch version
+    echo - test-yaml: Generates and runs yaml_test_runner package xpack/oss tests against a given Elasticsearch version
+    echo - test-generator: Generates and runs api_generator package tests
+    echo - test: Runs elasticsearch package tests against a given Elasticsearch version
     echo.
     echo Most tasks use these environment variables:
     echo - STACK_VERSION (default '$STACK_VERSION'): the version of Elasticsearch
     echo - TEST_SUITE ('oss' or 'xpack', default '$TEST_SUITE'): the distribution of Elasticsearch
+    echo - CI (default not set): set when running on CI to determine whether to start Elasticsearch and format test output as JSON
     echo.
     echo Run 'cargo make --list-all-steps' for a complete list of available tasks.
     echo.

elasticsearch/examples/cat_indices.rs

@@ -45,7 +45,7 @@ pub async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
 fn create_client() -> Result<Elasticsearch, Error> {
     fn cluster_addr() -> String {
-        match std::env::var("ES_TEST_SERVER") {
+        match std::env::var("ELASTICSEARCH_URL") {
             Ok(server) => server,
             Err(_) => DEFAULT_ADDRESS.into(),
         }

elasticsearch/examples/index_questions_answers/main.rs

@@ -362,7 +362,7 @@ async fn create_index_if_not_exists(client: &Elasticsearch, delete: bool) -> Res
 
 fn create_client() -> Result<Elasticsearch, Error> {
     fn cluster_addr() -> String {
-        match std::env::var("ES_TEST_SERVER") {
+        match std::env::var("ELASTICSEARCH_URL") {
             Ok(server) => server,
             Err(_) => DEFAULT_ADDRESS.into(),
         }

elasticsearch/examples/search_questions/main.rs

@@ -112,7 +112,7 @@ pub async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
 fn create_client() -> Result<Elasticsearch, Error> {
     fn cluster_addr() -> String {
-        match std::env::var("ES_TEST_SERVER") {
+        match std::env::var("ELASTICSEARCH_URL") {
             Ok(server) => server,
             Err(_) => DEFAULT_ADDRESS.into(),
         }

elasticsearch/examples/search_questions_answers/main.rs

@@ -99,7 +99,7 @@ pub async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
 fn create_client() -> Result<Elasticsearch, Error> {
     fn cluster_addr() -> String {
-        match std::env::var("ES_TEST_SERVER") {
+        match std::env::var("ELASTICSEARCH_URL") {
             Ok(server) => server,
             Err(_) => DEFAULT_ADDRESS.into(),
         }

elasticsearch/tests/common/client.rs

@@ -36,7 +36,7 @@ use url::Url;
 /// Gets the address to the Elasticsearch instance from environment variables
 /// and assumes an instance running locally on the default port otherwise
 pub fn cluster_addr() -> String {
-    match std::env::var("ES_TEST_SERVER") {
+    match std::env::var("ELASTICSEARCH_URL") {
         Ok(server) => server,
         Err(_) => DEFAULT_ADDRESS.into(),
     }

yaml_test_runner/tests/common/client.rs

@@ -53,7 +53,7 @@ use sysinfo::SystemExt;
 use url::Url;
 
 fn cluster_addr() -> String {
-    match std::env::var("ES_TEST_SERVER") {
+    match std::env::var("ELASTICSEARCH_URL") {
         Ok(server) => server,
         Err(_) => DEFAULT_ADDRESS.into(),
     }