Run integration tests in CI

.github/workflows/push.yaml

@@ -11,19 +11,30 @@ jobs:
   test:
     name: Run tests
     runs-on: ubuntu-latest
+    services:
+      elasticsearch:
+        image: docker.elastic.co/elasticsearch/elasticsearch:7.9.3
+        env:
+          discovery.type: single-node
+        ports:
+          - 9200:9200
+          - 9300:9300
     steps:
-      - uses: actions/checkout@v1
-
-      - uses: actions/setup-node@master
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
         with:
-          node-version: '14.18'
-
-      - run: npm install -g npm@latest
-
-      - run: npm install
-
-      - run: npm run lint
-
-      - run: npm audit --omit dev
-
-      - run: npm run test:coverage
+          node-version-file: ".nvmrc"
+          cache: npm
+          cache-dependency-path: package.json
+      - name: Upgrade npm
+        run: npm install -g npm@latest
+      - name: Install dependencies
+        run: npm install
+      - name: Eslint
+        run: npm run lint
+      - name: Audit
+        run: npm audit --omit dev
+      - name: Unit tests
+        run: npm run test:coverage
+      - name: Integration tests
+        run: npm run test:integration

README.md

@@ -12,8 +12,9 @@
     - [Subscribing to SNS Topics](#subscribing-to-sns-topics)
     - [Ingest Errors](#ingest-errors)
   - [Development](#development)
-    - [Running Locally using AWS Resources](#running-locally-using-aws-resources)
-    - [Running Locally using Local Resources](#running-locally-using-local-resources)
+    - [Running Locally](#running-locally)
+    - [Running Tests](#running-tests)
+    - [Integration Tests](#integration-tests)
   - [About](#about)
 
 ## Overview
@@ -158,27 +159,35 @@ npm run test
 npm run build-api-docs # TODO: this fails
 ```
 
-### Running Locally using AWS Resources
+### Running Locally
 
-The easiest way to run the API server locally is to deploy an instance to AWS, and then run a local instance pointing at the Elasticsearch resource created by that deployment. 
+The easiest way to run the API server locally is to use an Elasticsearch container running in Docker and `serverless-offline`.
 
-To do this, first change the value of `ES_HOST` in the serverless.yml file from the default (which dynamically populates it from the resource created upon deploy):
+There is a `docker-compose.yml` file to simplify running Elasticsearch locally:
 
+```sh
+docker-compose up -d
 ```
+
+The API can be run using `serverless-offline`, which will require a `serverless.yml` file. Copy `serverless.yml.example` to `serverless.yml`. You'll also need to edit the `serverless.yml` file to set the `ES_HOST` value.
+
+To do this, first change the value of `ES_HOST` in the serverless.yml file from the default (which dynamically populates it from the resource created upon deploy):
+
+```yml
 ES_HOST:
   Fn::GetAtt: [ElasticSearchInstance, DomainEndpoint]
 ```
 
 to instead use a hard-coded string of the Elasticsearch instance URL, e.g.,
 
-```
-ES_HOST: https://search-stac-server-dev-es-7bwzvxndbxp4dl6h344xlpvly.us-west-2.es.amazonaws.com/
+```yml
+ES_HOST: http://localhost:9200
 ```
 
 also set the STAC_API_URL explicitly:
 
-```
-STAC_API_URL: http://localhost:3000
+```yml
+STAC_API_URL: http://localhost:3000/dev
 ```
 
 Then, use npm `serve` command to run serverless offline:
@@ -198,13 +207,6 @@ curl -s https://planetarycomputer.microsoft.com/api/stac/v1/collections/aster-l1
     aws sns publish --topic-arn ${TOPIC_ARN} --message file:///dev/stdin
 ```
 
-### Running Locally using Local Resources
-
-TBD, possibly using:
-
-- https://github.com/localstack/localstack
-- https://github.com/localstack/serverless-localstack
-
 ## Running Tests
 
 stac-server uses [ava](https://github.com/avajs/ava) to execute tests.
@@ -219,13 +221,41 @@ npm run test:unit
 # run unit tests with coverage
 npm run test:coverage
 
-# run integration tests
-npm run test:integration
-
 # run tests from a single test file whose titles match 'foobar*'
 npx ava tests/test_es.js --match='foobar*'
 ```
 
+### Integration Tests
+
+The integration tests use an Elasticsearch server running in Docker and an instance of the API using [Serverless Offline](https://www.npmjs.com/package/serverless-offline).
+
+When the integration tests run, they:
+
+1. Wait for Elasticsearch to be available
+1. Delete all indices from Elasticsearch
+1. Add indices and test data to Elasticsearch
+1. Move any existing `serverless.yml` file to `serverless.yml.original`
+1. Create a `serverless.yml` file to support serverless-offline
+1. Build the packages
+1. Start an instance of the API using serverless-offline. That API will be available at <http://localhost:3000/dev/>
+1. Wait for the API to be available
+1. Run the integration tests in `./tests/integration/test_*.js`
+1. Remove the serverless.yml file
+1. Restore `serverless.yml.original` to `serverless.yml`, if necessary
+1. Stop the API
+
+Before running the integration tests, make sure to start Elasticsearch using:
+
+```sh
+docker-compose up -d
+```
+
+Once Elasticsearch has been started, run the integration tests:
+
+```sh
+npm run test:integration
+```
+
 ## About
 
 [stac-server](https://github.com/stac-utils/stac-server) was forked from [sat-api](https://github.com/sat-utils/sat-api). Stac-server is for STAC versions 0.9.0+, while sat-api exists for versions of STAC prior to 0.9.0.

bin/integration-tests.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+set -e
+
+export ES_HOST='http://localhost:9200'
+export AWS_ACCESS_KEY_ID='none'
+export AWS_SECRET_ACCESS_KEY='none'
+
+./bin/wait-for-elasticsearch/run.sh
+
+echo "Setting up Elasticsearch"
+node ./tests/integration/setup-es.js
+
+echo "Configuring serverless-offline"
+if [ -e serverless.yml ]; then
+  mv serverless.yml serverless.yml.original
+fi
+
+grep -v 'ElasticSearchInstance, DomainEndpoint' serverless.yml.example |\
+  sed 's/ES_HOST.*/ES_HOST: http:\/\/localhost:9200\n    STAC_API_URL: http:\/\/localhost:3000\/dev/' > serverless.yml
+
+echo "Building packages"
+npm run build
+
+echo "Starting serverless-offline"
+npx serverless offline start >/dev/null 2>&2 &
+SERVERLESS_PID="$!"
+
+./bin/wait-for-serverless-offline/run.sh
+
+echo "Running tests"
+set +e
+npx ava ./tests/integration/test_*.js
+TEST_RESULT="$?"
+set -e
+
+rm -f serverless.yml
+
+if [ -e serverless.yml.original ]; then
+  mv serverless.yml.original serverless.yml
+fi
+
+echo "Stopping serverless-offline"
+kill "$SERVERLESS_PID"
+
+exit "$TEST_RESULT"

bin/wait-for-elasticsearch/run.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -e
+
+D=$(dirname "$0")
+
+echo 'Waiting for ElasticSearch'
+
+timeout 60 "${D}/wait.sh"
+
+echo 'Elasticsearch is up'

bin/wait-for-elasticsearch/wait.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+is_es_up () {
+  curl -s -X GET "http://127.0.0.1:9200/_cluster/health" |\
+    grep -E '"status":"(green|yellow)"' > /dev/null 2>&1
+}
+
+while ! is_es_up; do sleep 1; done

bin/wait-for-serverless-offline/run.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -e
+
+D=$(dirname "$0")
+
+echo 'Waiting for Serverless Offline'
+
+timeout 60 "${D}/wait.sh"
+
+echo 'Serverless Offline has started'

bin/wait-for-serverless-offline/wait.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+is_serverless_offline_running () {
+  curl -s -X GET "http://127.0.0.1:3000/dev/" > /dev/null 2>&1
+}
+
+while ! is_serverless_offline_running; do sleep 1; done

docker-compose.yml

@@ -0,0 +1,9 @@
+version: "3.8"
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.9.3
+    ports:
+      - "127.0.0.1:9200:9200"
+      - "127.0.0.1:9300:9300"
+    environment:
+      - discovery.type=single-node

libs/es.js

@@ -245,6 +245,7 @@ async function editPartialItem(itemId, updateFields) {
 async function esQuery(parameters) {
   logger.info(`Elasticsearch query: ${JSON.stringify(parameters)}`)
   const client = await esClient.client()
+  if (client === undefined) throw new Error('Client is undefined')
   const response = await client.search(parameters)
   logger.info(`Response: ${JSON.stringify(response)}`)
   return response

libs/esClient.js

@@ -18,7 +18,7 @@ async function connect() {
   // use local client
   if (!process.env.ES_HOST) {
     esConfig = {
-      node: 'localhost:9200'
+      node: 'http://localhost:9200'
     }
     client = new elasticsearch.Client(esConfig)
   } else {

libs/esStream.js

@@ -80,7 +80,7 @@ class ElasticSearchWritableStream extends _stream.Writable {
     const body = this.transformRecords(records)
     try {
       const result = await this.client.bulk({ body })
-      logger.debug(`Result: ${result}`)
+      logger.debug(`Result: ${JSON.stringify(result, undefined, 2)}`)
       const { errors, items } = result.body
       if (errors) {
         logger.error(items)