Merge branch 'release/0.11.0'

snowplow · Nov 4, 2020 · 0a5d907 · 0a5d907
2 parents 8767b9e + be13e40
commit 0a5d907
Show file tree

Hide file tree

Showing 15 changed files with 106 additions and 89 deletions.
diff --git a/.travis/deploy.sh → .github/workflows/deploy.sh b/.travis/deploy.sh → .github/workflows/deploy.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 
-cd $TRAVIS_BUILD_DIR
+cd $GITHUB_WORKSPACE
 
 ###### input validation ######
 if [ "$#" -ne 1 ]
@@ -18,7 +18,7 @@ else
 		echo "Platform recognized!"
 	else
 		echo "Unrecognized platform! Aborted."
-		echo "Supported platforms; aws, gcp."
+		echo "Supported platforms: aws, gcp."
 		exit 1
 	fi
 
@@ -72,7 +72,7 @@ else
 		fi
 	else
 		echo "Unrecognized size! Aborted."
-		echo "Available sizes; large, xlarge and xxlarge."
+		echo "Available sizes: large, xlarge and xxlarge."
 		exit 1
 	fi
 fi
@@ -86,6 +86,8 @@ then
 	packer build -only=amazon-ebs Packerfile.json
 elif [ "$SP_MINI_PLATFORM" == "gcp" ]
 then
+	echo $GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64 | base64 --decode > $GITHUB_WORKSPACE/BIN
+	export GOOGLE_APPLICATION_CREDENTIALS=$GITHUB_WORKSPACE/BIN
 	packer build -only=googlecompute Packerfile.json
 else
 	echo "Unrecognized platform. Aborted."

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -0,0 +1,56 @@
+name: Test and Publish Mini
+
+on:
+  push:
+
+jobs:
+  test:
+    name: Test and publish artifacts
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Setup Go 1.14.10
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.14.10'
+
+      - name: Check Go version
+        run: go version
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y language-pack-en python-pip libffi-dev libssl-dev python-dev
+          sudo pip install --upgrade pip
+          sudo pip install markupsafe==1.1.1 setuptools==40.8.0 ansible==2.8.1 paramiko==1.16.0
+
+      - name: Provision a local Mini instance
+        run: ansible-playbook -i $GITHUB_WORKSPACE/provisioning/inventory $GITHUB_WORKSPACE/provisioning/local_setup.yml --connection=local --become
+
+      - name: Integration test
+        run: $GITHUB_WORKSPACE/integration/integration_test.sh
+
+      - name: Set up gcloud
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+        with:
+          version: '290.0.1'
+          service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64 }}
+          export_default_credentials: true
+      - run: gcloud info
+
+      - name: Extract tag from ref
+        if: startsWith(github.ref, 'refs/tags/')
+        id: get_tag
+        run: echo ::set-output name=VERSION::$(echo $GITHUB_REF | sed "s/refs\/tags\///")
+
+      - name: Publish artifact based on cloud and size
+        if: startsWith(github.ref, 'refs/tags/')
+        env:
+          AWS_DEPLOY_ACCESS_KEY: ${{ secrets.AWS_DEPLOY_ACCESS_KEY }}
+          AWS_DEPLOY_SECRET_KEY: ${{ secrets.AWS_DEPLOY_SECRET_KEY }}
+          GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64 }}
+        run: $GITHUB_WORKSPACE/.github/workflows/deploy.sh ${{ steps.get_tag.outputs.VERSION }}
diff --git a/.travis.yml b/.travis.yml
diff --git a/.travis/is_valid_tag.sh b/.travis/is_valid_tag.sh
diff --git a/BIN.enc b/BIN.enc
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,3 +1,13 @@
+Version 0.11.0 (2020-11-04)
+---------------------------
+Migrate from Travis to GitHub Actions (#250)
+Remove Authorization header (#245)
+Update documentation on index page (#247)
+Bump Enrich to 1.4.1 (#243)
+Bump Stream Collector to 2.0.0 (#241)
+Disable default redirect path in collector configuration (#239)
+Add section about errors when building with Vagrant to README.md (#238)
+
 Version 0.10.1 (2020-09-07)
 ---------------------------
 Bump Enrich to 1.3.2 (#240)

diff --git a/Packerfile.json b/Packerfile.json
@@ -57,8 +57,8 @@
     }
   ],
   "variables": {
-    "aws_version": "0.10.1",
-    "gcp_version": "0-10-1",
+    "aws_version": "0.11.0",
+    "gcp_version": "0-11-0",
     "gcp_image_name": "snowplow-mini-{{user `gcp_version`}}-{{user `sp_mini_size`}}-{{timestamp}}",
     "gcp_machine_type": "{{env `GCP_MACHINE_TYPE`}}",
     "aws_instance_type": "{{env `AWS_INSTANCE_TYPE`}}",

diff --git a/README.md b/README.md
@@ -55,6 +55,20 @@ These events can then be viewed in Kibana at `http://< sp mini public ip>/kibana
 
 Documentation is available at [our docs website][mini-docs].
 
+## Vagrant
+
+Some advice on how to handle certain errors if you're trying to build this locally with Vagrant.
+
+### `The box 'ubuntu/xenial64' could not be found or could not be accessed in the remote catalog.`
+
+Your Vagrant version is probably outdated. Use Vagrant 2.0.0+.
+
+### `npm install` results in `enoent ENOENT: no such file or directory, open '/package.json'`
+
+This is caused by trying to use NFS. Comment the relevant lines in `Vagrantfile`.
+
+Most likely this will happen on `TASK [sp_mini_5_build_ui : Install npm packages based on package.json.]` but see also: https://discourse.snowplowanalytics.com/t/snowplow-mini-local-vagrant/2930.
+
 ## Copyright and license
 
 Snowplow Mini is copyright 2016-2020 Snowplow Analytics Ltd.
@@ -73,7 +87,7 @@ limitations under the License.
 [travis]: https://travis-ci.org/snowplow/snowplow-mini
 [travis-image]: https://travis-ci.org/snowplow/snowplow-mini.svg?branch=master
 
-[release-image]: http://img.shields.io/badge/release-0.10.1-blue.svg?style=flat
+[release-image]: http://img.shields.io/badge/release-0.11.0-blue.svg?style=flat
 [releases]: https://github.com/snowplow/snowplow-mini/releases
 
 [license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.10.1
+0.11.0
diff --git a/integration/integration_test.sh b/integration/integration_test.sh
@@ -1,13 +1,14 @@
 #!/bin/bash
 
-# Send good and bad events
+# Send good and bad events, and test that the default redirect path is disabled
 COUNTER=0
 while [  $COUNTER -lt 10 ]; do
   curl http://localhost:8080/i?e=pv
   curl http://localhost:8080/i
+  curl http://localhost:8080/r/tp2
   let COUNTER=COUNTER+1
 done
-sleep 120
+sleep 30
 
 # Assertions
 good_count="$(curl --silent -XGET 'http://localhost:9200/good/good/_count' | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["count"]')"

diff --git a/provisioning/resources/configs/Caddyfile b/provisioning/resources/configs/Caddyfile
@@ -17,7 +17,9 @@
   redir /nsqadmin /nsqadmin/
   redir /elasticsearch /elasticsearch/
 
-  proxy / localhost:8080
+  proxy / localhost:8080 {
+    header_upstream -Authorization
+  }
 
   proxy /home localhost:3000 {
     without /home

diff --git a/provisioning/resources/configs/snowplow-stream-collector.hocon b/provisioning/resources/configs/snowplow-stream-collector.hocon
@@ -64,6 +64,8 @@ collector {
     body = "302, redirecting"
   }
 
+  enableDefaultRedirect = false
+
   redirectMacro {
     enabled = false
     placeholder = "[TOKEN]"

diff --git a/provisioning/resources/ui/index.html b/provisioning/resources/ui/index.html
@@ -14,7 +14,7 @@ <h1 id="text">SNOWPLOW MINI</h1>
     </header>
     <div class="snowplowminiapp"></div>
     <footer class="footer">
-      <p>COPYRIGHT &#169; 2016-2018 SNOWPLOW ANALYTICS LTD. ALL RIGHTS RESERVED. PRIVACY POLICY.</p>
+      <p>COPYRIGHT &#169; 2016-2020 SNOWPLOW ANALYTICS LTD. ALL RIGHTS RESERVED. PRIVACY POLICY.</p>
     </footer>
 
     <!-- Snowplow starts plowing -->

diff --git a/provisioning/resources/ui/js/components/Overview.tsx b/provisioning/resources/ui/js/components/Overview.tsx
@@ -42,14 +42,17 @@ export class Overview extends React.Component<{}, {}> {
         <h3>3. Understanding how Snowplow Mini works</h3>
         <h3>Quicklinks: </h3>
         <ul>
+          <li><a href={'https://docs.snowplowanalytics.com/docs/understanding-your-pipeline/what-is-snowplow-mini'}>What is Snowplow Mini?</a></li>
+          <li><a href={'https://docs.snowplowanalytics.com/docs/open-source-components-and-applications/snowplow-mini/snowplow-mini-0-11-0/usage-guide/'}>Usage guide</a></li>
+          <li><a href={'https://docs.snowplowanalytics.com/docs/open-source-components-and-applications/snowplow-mini/snowplow-mini-0-11-0/control-plane-api/'}>Control Plane API</a></li>
           <li>Link to <a href={'https://github.com/snowplow/snowplow-mini'}>Snowplow Mini</a> repository</li>
-          <li>Link to <a href={'https://github.com/snowplow/snowplow-mini/wiki/Quickstart-guide'}>Quickstart Guide</a></li>
           <li>Collector endpoint <a href={collector}>{collector}</a></li>
         </ul>
         <h3>The software stack installed: </h3>
         <ul>
-          <li>Snowplow Stream Collector NSQ 1.0.1</li>
-          <li>Snowplow Stream Enrich NSQ 1.3.2</li>
+        <li><b>Snowplow Mini 0.11.0</b></li>
+          <li>Snowplow Stream Collector NSQ 2.0.0</li>
+          <li>Snowplow Stream Enrich NSQ 1.4.1</li>
           <li>Snowplow Elasticsearch Loader 1.0.0</li>
           <li>Snowplow Iglu Server 0.6.1</li>
           <li>Postgres 9.5</li>

diff --git a/provisioning/roles/docker/files/docker-compose.yml b/provisioning/roles/docker/files/docker-compose.yml
@@ -108,7 +108,7 @@ services:
       - "4171:4171"
 
   scala-stream-collector:
-    image: snowplow/scala-stream-collector-nsq:1.0.1
+    image: snowplow/scala-stream-collector-nsq:2.0.0
     container_name: scala-stream-collector-nsq
     command: [ "--config", "/snowplow/config/snowplow-stream-collector.hocon" ]
     restart: always
@@ -126,7 +126,7 @@ services:
       - "SP_JAVA_OPTS=-Xmx${SP_JVM_SIZE}"
 
   stream-enrich:
-    image: snowplow/stream-enrich-nsq:1.3.2
+    image: snowplow/stream-enrich-nsq:1.4.1
     container_name: stream-enrich-nsq
     command: [
       "--config", "/snowplow/config/snowplow-stream-enrich.hocon",