Run rest-api-spec / YAML tests in CI (#303)

* Run yaml-test Signed-off-by: Thomas Farr <[email protected]> * Fixes Signed-off-by: Thomas Farr <[email protected]> * Separate workflow Signed-off-by: Thomas Farr <[email protected]> --------- Signed-off-by: Thomas Farr <[email protected]>
opensearch-project · Aug 7, 2023 · dcf543f · dcf543f
1 parent b5728bb
commit dcf543f
Show file tree

Hide file tree

Showing 18 changed files with 736 additions and 423 deletions.
diff --git a/.ci/certs/openssl.conf b/.ci/certs/openssl.conf
@@ -0,0 +1,33 @@
+[req]
+distinguished_name = req_distinguished_name
+
+[req_distinguished_name]
+
+[root-ca]
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid:always
+basicConstraints = critical, CA:TRUE
+keyUsage = critical, digitalSignature, keyCertSign, cRLSign
+
+[esnode]
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid:always, issuer:always
+basicConstraints = critical, CA:FALSE
+keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment
+extendedKeyUsage = critical, serverAuth, clientAuth
+subjectAltName = @esnode-san
+
+[esnode-san]
+DNS.1 = localhost
+DNS.2 = instance
+DNS.3 = instance1
+DNS.4 = instance2
+IP.1 = 127.0.0.1
+IP.2 = 0:0:0:0:0:0:0:1
+
+[kirk]
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid:always, issuer:always
+basicConstraints = critical, CA:FALSE
+keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment
+extendedKeyUsage = critical, clientAuth
diff --git a/.ci/generate-certs.sh b/.ci/generate-certs.sh
@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+script_path=$(dirname $(realpath -s $0))
+certs_dir="$script_path/certs"
+opensearch_dir="$script_path/opensearch"
+
+openssl_conf="$certs_dir/openssl.conf"
+
+root_ca_key="$certs_dir/root-ca.key"
+root_ca_crt="$certs_dir/root-ca.crt"
+root_ca_pem="$opensearch_dir/root-ca.pem"
+
+esnode_key="$certs_dir/esnode.key"
+esnode_key_pem="$opensearch_dir/esnode-key.pem"
+esnode_csr="$certs_dir/esnode.csr"
+esnode_crt="$certs_dir/esnode.crt"
+esnode_pem="$opensearch_dir/esnode.pem"
+
+kirk_key="$certs_dir/kirk.key"
+kirk_csr="$certs_dir/kirk.csr"
+kirk_crt="$certs_dir/kirk.crt"
+kirk_p12="$certs_dir/kirk.p12"
+
+common_crt_args="-extfile $openssl_conf -days 36500 -CA $root_ca_crt -CAkey $root_ca_key -CAcreateserial"
+common_csr_args="-config $openssl_conf -days 36500"
+
+# Stop Git Bash / MSYS / Cygwin from mangling the cert subjects
+subj_prefix=""
+if [[ "$(uname)" == MINGW* ]]; then
+    subj_prefix="/"
+fi
+
+if [[ ! -f $root_ca_key ]]; then
+    rm -f $root_ca_crt
+    openssl genrsa -out $root_ca_key
+fi
+
+if [[ ! -f $root_ca_crt ]]; then
+    rm -f *.crt $root_ca_pem
+    openssl req -new -x509 \
+        -key $root_ca_key \
+        -subj "$subj_prefix/DC=com/DC=example/O=Example Com Inc./OU=Example Com Inc. Root CA/CN=Example Com Inc. Root CA" \
+        $common_csr_args -extensions root-ca \
+        -out $root_ca_crt
+fi
+
+if [[ ! -f $root_ca_pem ]]; then
+    cp $root_ca_crt $root_ca_pem
+fi
+
+if [[ ! -f $esnode_key ]]; then
+    rm -f $esnode_csr $esnode_key_pem
+    openssl genrsa -out $esnode_key
+fi
+
+if [[ ! -f $esnode_key_pem ]]; then
+    openssl pkcs8 -topk8 -in $esnode_key -nocrypt -out $esnode_key_pem
+fi
+
+if [[ ! -f $esnode_csr ]]; then
+    rm -f $esnode_crt
+    openssl req -new \
+        $common_csr_args \
+        -key $esnode_key \
+        -subj "$subj_prefix/DC=de/L=test/O=node/OU=node/CN=node-0.example.com" \
+        -out $esnode_csr
+fi
+
+if [[ ! -f $esnode_crt ]]; then
+    rm -f $esnode_pem
+    openssl x509 -req -in $esnode_csr $common_crt_args -extensions esnode -out $esnode_crt
+fi
+
+if [[ ! -f $esnode_pem ]]; then
+    cp $esnode_crt $esnode_pem
+fi
+
+if [[ ! -f $kirk_key ]]; then
+    rm -f $kirk_csr
+    openssl genrsa -out $kirk_key
+fi
+
+if [[ ! -f $kirk_csr ]]; then
+    rm -f $kirk_crt
+    openssl req -new \
+        $common_csr_args \
+        -subj "$subj_prefix/C=de/L=test/O=client/OU=client/CN=kirk" \
+        -key $kirk_key \
+        -out $kirk_csr
+fi
+
+if [[ ! -f $kirk_crt ]]; then
+    rm -f $kirk_p12
+    openssl x509 -req -in $kirk_csr $common_crt_args -extensions kirk -out $kirk_crt
+fi
+
+if [[ ! -f $kirk_p12 ]]; then
+    openssl pkcs12 -export \
+        -in $kirk_crt \
+        -inkey $kirk_key \
+        -descert \
+        -passout pass:kirk \
+        -out $kirk_p12
+fi
diff --git a/.ci/opensearch/opensearch.yml b/.ci/opensearch/opensearch.yml
@@ -0,0 +1,11 @@
+network.host: 0.0.0.0
+node.name: instance
+cluster.name: search-rest-test
+cluster.initial_master_nodes: instance
+discovery.seed_hosts: instance
+cluster.routing.allocation.disk.threshold_enabled: false
+bootstrap.memory_lock: true
+node.attr.testattr: test
+path.repo: /tmp
+repositories.url.allowed_urls: http://snapshot.test*
+action.destructive_requires_name: false
diff --git a/.github/actions/build-opensearch/action.yml b/.github/actions/build-opensearch/action.yml
@@ -0,0 +1,103 @@
+name: Restore or Build OpenSearch
+description: Restore or Build OpenSearch from source
+inputs:
+  ref:
+    description: The git ref to clone
+  build_snapshot:
+    description: Whether to build a snapshot version
+    default: "true"
+  security_plugin:
+    description: Whether to build the security plugin
+    default: "false"
+  knn_plugin:
+    description: Whether to build the k-nn plugin
+    default: "false"
+  plugins_output_directory:
+    description: The directory to output the plugins to
+    default: ""
+outputs: 
+    distribution:
+      description: The path to the OpenSearch distribution
+      value: ${{ steps.determine.outputs.distribution }}
+    version:
+      description: The version of OpenSearch
+      value: ${{ steps.determine.outputs.version }}
+runs:
+  using: composite
+  steps:
+    - name: Restore or Build OpenSearch
+      uses: ./client/.github/actions/cached-git-build
+      with:
+        repository: opensearch-project/OpenSearch
+        ref: ${{ inputs.ref }}
+        path: opensearch
+        cache_key_suffix: ${{ inputs.build_snapshot == 'true' && '-snapshot' || '' }}
+        cached_paths: |
+          ./opensearch/distribution/archives/linux-tar/build/distributions/opensearch-*.tar.gz
+          ./opensearch/plugins/*/build/distributions/*.zip
+        build_script: |
+          ./gradlew :distribution:archives:linux-tar:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }}
+
+          PluginList=("analysis-icu"  "analysis-kuromoji"  "analysis-nori"  "analysis-phonetic"  "ingest-attachment"  "mapper-murmur3")
+          for plugin in ${PluginList[*]}; do
+            ./gradlew :plugins:$plugin:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }}
+          done
+
+    - name: Determine OpenSearch distribution path and version
+      id: determine
+      shell: bash -eo pipefail {0}
+      run: |
+        distribution=`ls -1 $PWD/opensearch/distribution/archives/linux-tar/build/distributions/opensearch-*.tar.gz | head -1`
+        version=`basename $distribution | cut -d'-' -f3,${{ inputs.build_snapshot == 'true' && 4 || 3 }}`
+        echo "distribution=$distribution" | tee -a $GITHUB_OUTPUT
+        echo "version=$version" | tee -a $GITHUB_OUTPUT
+
+    - name: Restore or Build OpenSearch Security
+      uses: ./client/.github/actions/cached-git-build
+      if: inputs.security_plugin == 'true'
+      with:
+        repository: opensearch-project/security
+        ref: ${{ inputs.ref }}
+        path: opensearch-security
+        cache_key_suffix: ${{ inputs.build_snapshot == 'true' && '-snapshot' || '' }}
+        cached_paths: |
+          ./opensearch-security/build/distributions/opensearch-security-*.zip
+        build_script: ./gradlew assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }}
+
+    - name: Restore or Build OpenSearch k-NN
+      uses: ./client/.github/actions/cached-git-build
+      if: inputs.knn_plugin == 'true'
+      with:
+        repository: opensearch-project/k-NN
+        ref: ${{ inputs.ref }}
+        path: opensearch-knn
+        cache_key_suffix: ${{ inputs.build_snapshot == 'true' && '-snapshot' || '' }}
+        cached_paths: |
+          ./opensearch-knn/build/distributions/opensearch-knn-*.zip
+        build_script: |
+          sudo apt-get install -y libopenblas-dev libomp-dev
+          ./gradlew buildJniLib assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }}
+          distributions=./build/distributions
+          lib_dir=$distributions/lib
+          mkdir $lib_dir
+          cp -v $(ldconfig -p | grep libgomp | cut -d ' ' -f 4) $lib_dir
+          cp -v ./jni/release/libopensearchknn_* $lib_dir
+          ls -l $lib_dir
+          cd $distributions
+          zip -ur opensearch-knn-*.zip lib
+
+    - name: Copy OpenSearch plugins
+      shell: bash -eo pipefail {0}
+      if: inputs.plugins_output_directory != ''
+      run: |
+        mkdir -p ${{ inputs.plugins_output_directory }}
+        cp -v ./opensearch/plugins/*/build/distributions/*.zip ${{ inputs.plugins_output_directory }}/
+
+        plugins=("opensearch-knn" "opensearch-security")
+        for plugin in ${plugins[*]}; do
+          if [[ -d "./$plugin" ]]; then
+            cp -v ./$plugin/build/distributions/$plugin-*.zip ${{ inputs.plugins_output_directory }}/
+          fi
+        done
+
+        ls -l ${{ inputs.plugins_output_directory }}
diff --git a/.github/actions/cached-git-build/action.yml b/.github/actions/cached-git-build/action.yml
@@ -11,6 +11,9 @@ inputs:
     description: A list of paths to cache
   build_script:
     description: The script to run to build the repository
+  cache_key_suffix:
+    description: A suffix to append to the cache key
+    default: ''
 runs:
   using: composite
   steps:
@@ -32,7 +35,7 @@ runs:
       uses: actions/cache/restore@v3
       with:
         path: ${{ inputs.cached_paths }}
-        key: ${{ inputs.repository }}-${{ steps.get-sha.outputs.sha }}
+        key: ${{ inputs.repository }}-${{ steps.get-sha.outputs.sha }}${{ inputs.cache_key_suffix }}
 
     - name: Build
       if: steps.restore.outputs.cache-hit != 'true'
@@ -45,4 +48,4 @@ runs:
       uses: actions/cache/save@v3
       with:
         path: ${{ inputs.cached_paths }}
-        key: ${{ inputs.repository }}-${{ steps.get-sha.outputs.sha }}
+        key: ${{ inputs.repository }}-${{ steps.get-sha.outputs.sha }}${{ inputs.cache_key_suffix }}
diff --git a/.github/actions/run-released-opensearch/action.yml b/.github/actions/run-released-opensearch/action.yml
@@ -0,0 +1,49 @@
+name: Run OpenSearch
+description: Runs a released version of OpenSearch
+inputs:
+  version:
+    description: The version of OpenSearch to run
+    required: true
+  secured:
+    description: Whether to enable the security plugin
+    required: true
+outputs:
+  opensearch_url:
+    description: The URL where the OpenSearch node is accessible
+    value: ${{ steps.opensearch.outputs.opensearch_url }}
+runs:
+  using: composite
+  steps:
+    - name: Restore cached OpenSearch distro
+      id: cache-restore
+      uses: actions/cache/restore@v3
+      with:
+        path: opensearch-*
+        key: opensearch-${{ inputs.version }}-${{ runner.os }}
+
+    - name: Download OpenSearch
+      if: steps.cache-restore.outputs.cache-hit != 'true'
+      shell: bash -eo pipefail {0}
+      run: |
+        if [[ "$RUNNER_OS" != "Windows" ]]; then
+          curl -sSLO https://artifacts.opensearch.org/releases/bundle/opensearch/${{ inputs.version }}/opensearch-${{ inputs.version }}-linux-x64.tar.gz
+          tar -xzf opensearch-*.tar.gz
+          rm -f opensearch-*.tar.gz
+        else
+          curl -sSLO https://artifacts.opensearch.org/releases/bundle/opensearch/${{ inputs.version }}/opensearch-${{ inputs.version }}-windows-x64.zip
+          unzip opensearch-*.zip
+          rm -f opensearch-*.zip
+        fi
+
+    - name: Save cached OpenSearch distro
+      if: steps.cache-restore.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v3
+      with:
+        path: opensearch-*
+        key: opensearch-${{ inputs.version }}-${{ runner.os }}
+
+    - name: Start OpenSearch
+      id: opensearch
+      uses: ./client/.github/actions/start-opensearch
+      with:
+        secured: ${{ inputs.secured }}