diff --git a/.travis/deploy.sh b/.github/workflows/deploy.sh similarity index 90% rename from .travis/deploy.sh rename to .github/workflows/deploy.sh index 76338be2..06ab151d 100755 --- a/.travis/deploy.sh +++ b/.github/workflows/deploy.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -cd $TRAVIS_BUILD_DIR +cd $GITHUB_WORKSPACE ###### input validation ###### if [ "$#" -ne 1 ] @@ -18,7 +18,7 @@ else echo "Platform recognized!" else echo "Unrecognized platform! Aborted." - echo "Supported platforms; aws, gcp." + echo "Supported platforms: aws, gcp." exit 1 fi @@ -72,7 +72,7 @@ else fi else echo "Unrecognized size! Aborted." - echo "Available sizes; large, xlarge and xxlarge." + echo "Available sizes: large, xlarge and xxlarge." exit 1 fi fi @@ -86,6 +86,8 @@ then packer build -only=amazon-ebs Packerfile.json elif [ "$SP_MINI_PLATFORM" == "gcp" ] then + echo $GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64 | base64 --decode > $GITHUB_WORKSPACE/BIN + export GOOGLE_APPLICATION_CREDENTIALS=$GITHUB_WORKSPACE/BIN packer build -only=googlecompute Packerfile.json else echo "Unrecognized platform. Aborted." diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..b5f51869 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,56 @@ +name: Test and Publish Mini + +on: + push: + +jobs: + test: + name: Test and publish artifacts + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup Go 1.14.10 + uses: actions/setup-go@v2 + with: + go-version: '1.14.10' + + - name: Check Go version + run: go version + + - name: Install dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y language-pack-en python-pip libffi-dev libssl-dev python-dev + sudo pip install --upgrade pip + sudo pip install markupsafe==1.1.1 setuptools==40.8.0 ansible==2.8.1 paramiko==1.16.0 + + - name: Provision a local Mini instance + run: ansible-playbook -i $GITHUB_WORKSPACE/provisioning/inventory $GITHUB_WORKSPACE/provisioning/local_setup.yml --connection=local --become + + - name: Integration test + run: $GITHUB_WORKSPACE/integration/integration_test.sh + + - name: Set up gcloud + if: startsWith(github.ref, 'refs/tags/') + uses: GoogleCloudPlatform/github-actions/setup-gcloud@master + with: + version: '290.0.1' + service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64 }} + export_default_credentials: true + - run: gcloud info + + - name: Extract tag from ref + if: startsWith(github.ref, 'refs/tags/') + id: get_tag + run: echo ::set-output name=VERSION::$(echo $GITHUB_REF | sed "s/refs\/tags\///") + + - name: Publish artifact based on cloud and size + if: startsWith(github.ref, 'refs/tags/') + env: + AWS_DEPLOY_ACCESS_KEY: ${{ secrets.AWS_DEPLOY_ACCESS_KEY }} + AWS_DEPLOY_SECRET_KEY: ${{ secrets.AWS_DEPLOY_SECRET_KEY }} + GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64 }} + run: $GITHUB_WORKSPACE/.github/workflows/deploy.sh ${{ steps.get_tag.outputs.VERSION }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1c860757..00000000 --- a/.travis.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -sudo: required -dist: xenial -language: go - -go: 1.12 - -cache: - directories: - - "$HOME/google-cloud-sdk/" - -before_install: - - openssl aes-256-cbc -K $encrypted_44efaf791086_key -iv $encrypted_44efaf791086_iv -in BIN.enc -out BIN -d - - export GOOGLE_APPLICATION_CREDENTIALS=$TRAVIS_BUILD_DIR/BIN - - sudo apt-get update -qq - - sudo apt-get install -y language-pack-en python-pip python-paramiko=1.16.0-1ubuntu0.2 libffi-dev libssl-dev python-dev - - sudo pip install --upgrade pip - -install: - - sudo pip install markupsafe==1.1.1 - - sudo pip install setuptools==40.8.0 - - sudo pip install ansible==2.8.1 - - gcloud version || true - - if [ ! -d "$HOME/google-cloud-sdk/bin" ]; then rm -rf $HOME/google-cloud-sdk; export CLOUDSDK_CORE_DISABLE_PROMPTS=1; curl https://sdk.cloud.google.com | bash; fi - - source /home/travis/google-cloud-sdk/path.bash.inc # add the Google Cloud SDK command line tools (gsutil, gcloud) to PATH - - gcloud version - - gcloud -q components update - - if [ -a $TRAVIS_BUILD_DIR/BIN ]; then gcloud -q auth activate-service-account --key-file $TRAVIS_BUILD_DIR/BIN; fi - -before_script: - - ansible-playbook -i provisioning/inventory provisioning/local_setup.yml --connection=local --become - -script: ./integration/integration_test.sh - -deploy: - - provider: script - # travis builds timeout after 10 min of not seeing any output, hence the below work-around - # default travis_wait method doesn't show output until command finishes while this way allows us to see it - script: (while sleep 9m; do echo "=====[ $SECONDS seconds still running ]====="; done &) && (./.travis/deploy.sh $TRAVIS_TAG) - skip_cleanup: true - on: - condition: '"$(.travis/is_valid_tag.sh $TRAVIS_TAG)" == "" && $? == 0' - tags: true - -env: - global: - - secure: CP4KymCt5vhWEOGBYw+mBKxfd34PoGOybo5QdiGJ0tj5l6Tn8utPn/MSOMZLdlxdqUnPstnhsBNpGybnBSQIKC8TSZqMqgebw59Qq75bk2gNQ5NSv0iOjija9oKvBLaKgyXwwzL2o4JWnFTuFsHGyGLjSHoKy3sptfKIih3vwa0ey4SevElUtqpwT32AjCYW0L+iMP+kWRUEna6vdR4BTLrl8/UXAQRccQ8gmcpecfhqJL6sYgmK4oqgbhyMn8dvWVMIFsUjJ76rvJ5KvcGceNqJ07Yb8qaLKx/OBDn7I8GbRHrnxgLXF/7gU40jG+nOeYUTfdflMDmJTplk+Sgi2WxumpDdLuEJPge6kZMQGcZ5rHPS3dAXJ8APVn/SirHSVpjApdYkkoL7DXBFYPvIfpgKfqj2ofwKLoY4OZwOPU6+XCLQJkxxIszTPbyXaBofeB+CpLTIcU0nGn6NXAGT6ONDSlIKVNXotlhz9robtOeJ723tFgmUoTSvHHwXAal/3LBVjx2qnTg7pW3/9wcMaI+uaETdIcv3DBRpr8SfcObfYCMU7HOck3gfIIbn6YB8MbY3fceeZ8UPw9R9UyIvTVISOJW+SCCJoybEqxGnNhNXRTbn/anCS9dN1zesVCPZMl70bJyQNBkTW+sTusOASrfkYKvDiU/qkzUSx/Qlz0s= - - secure: gvyo8HuSuIEiSIZH7A2B0Nz6HJx04lTIV4QYx+bpEcCWailbgqSjDBRVwb6IdsQBsZsy79924Go3j4s0qtAOXGOLhHkmEE+bmLzT7Ejey9ENkQm5iAkKb2clYTKiDa94tB/Rqk+yW669HILttB/mcOAj+LFhBKb3jJJS0n0hDfYKsbleWSAVMyPw+V7SEQBc+9VTmJcq0YYYHS1Ie4g92QXxA33nLyGcOhCY7SS+gAqbMuQDhGa+Y0jh8Tz4YMVCd0tFLjsIB818DYE2YgRZerHUC8G7+kyIylV3/Nr1tEb/i6F8Ii6ET7GnzCnMJ0pj0vp/hsIsZBJeZUyUJEwtBx9798OeBWPrpuJtesTuNVO3DJmahrdIwe+fWOt5wFJFgZwvpmu12px+R8yTYkCl2oWSf+CNJi5fhb68m3D3p5JAqRNvvuKQE+zdX1pMCTmLWIjl8SOFN5ptP4kpDStziBsKyH9YQ8KUvR24kcyWO4u2pkRNgkI+SZTl83U/zPPKh/R1jGjm+TC9rlGCyj+OJIfl2hiLJKgBQtdSkJK0dz5ZQx9ZcGDPW88oeLUqJGRHQsPUVZB/cutrIwg2WIG4nBdCHhGG7t1u/hCr27aOTlJfy/3uA9cdkZCoVccwiwMcydtBmEgcYxjGbE378fXtBSpqPvuRs5r2dEpAbmcOmOU= diff --git a/.travis/is_valid_tag.sh b/.travis/is_valid_tag.sh deleted file mode 100755 index a287221b..00000000 --- a/.travis/is_valid_tag.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -e - -tag=$1 - -tag_version=`echo $tag | cut -d'/' -f1` -tag_platform=`echo $tag | cut -d'/' -f2` -tag_flavor=`echo $tag | cut -d'/' -f3` - -version=`cat ${TRAVIS_BUILD_DIR}/VERSION` - -if [ "${tag_version}" == "${version}" ]; then - if [ "${tag_platform}" == "aws" ] || [ "${tag_platform}" == "gcp" ]; then - if [ "${tag_flavor}" == "large" ] || [ "${tag_flavor}" == "xlarge" ] || [ "${tag_flavor}" == "xxlarge" ]; then - exit 0 - fi - echo "Tag flavor is not valid!" - exit 1 - fi - echo "Tag platform is not valid!" - exit 1 -else - echo "Tag version doesn't match VERSION file!" - exit 1 -fi diff --git a/BIN.enc b/BIN.enc deleted file mode 100644 index 624e076a..00000000 Binary files a/BIN.enc and /dev/null differ diff --git a/CHANGELOG b/CHANGELOG index b977b2ea..c94f2f6c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,13 @@ +Version 0.11.0 (2020-11-04) +--------------------------- +Migrate from Travis to GitHub Actions (#250) +Remove Authorization header (#245) +Update documentation on index page (#247) +Bump Enrich to 1.4.1 (#243) +Bump Stream Collector to 2.0.0 (#241) +Disable default redirect path in collector configuration (#239) +Add section about errors when building with Vagrant to README.md (#238) + Version 0.10.1 (2020-09-07) --------------------------- Bump Enrich to 1.3.2 (#240) diff --git a/Packerfile.json b/Packerfile.json index 1ac3dfd0..559e9766 100644 --- a/Packerfile.json +++ b/Packerfile.json @@ -57,8 +57,8 @@ } ], "variables": { - "aws_version": "0.10.1", - "gcp_version": "0-10-1", + "aws_version": "0.11.0", + "gcp_version": "0-11-0", "gcp_image_name": "snowplow-mini-{{user `gcp_version`}}-{{user `sp_mini_size`}}-{{timestamp}}", "gcp_machine_type": "{{env `GCP_MACHINE_TYPE`}}", "aws_instance_type": "{{env `AWS_INSTANCE_TYPE`}}", diff --git a/README.md b/README.md index 0eec5087..efef4c92 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,20 @@ These events can then be viewed in Kibana at `http://< sp mini public ip>/kibana Documentation is available at [our docs website][mini-docs]. +## Vagrant + +Some advice on how to handle certain errors if you're trying to build this locally with Vagrant. + +### `The box 'ubuntu/xenial64' could not be found or could not be accessed in the remote catalog.` + +Your Vagrant version is probably outdated. Use Vagrant 2.0.0+. + +### `npm install` results in `enoent ENOENT: no such file or directory, open '/package.json'` + +This is caused by trying to use NFS. Comment the relevant lines in `Vagrantfile`. + +Most likely this will happen on `TASK [sp_mini_5_build_ui : Install npm packages based on package.json.]` but see also: https://discourse.snowplowanalytics.com/t/snowplow-mini-local-vagrant/2930. + ## Copyright and license Snowplow Mini is copyright 2016-2020 Snowplow Analytics Ltd. @@ -73,7 +87,7 @@ limitations under the License. [travis]: https://travis-ci.org/snowplow/snowplow-mini [travis-image]: https://travis-ci.org/snowplow/snowplow-mini.svg?branch=master -[release-image]: http://img.shields.io/badge/release-0.10.1-blue.svg?style=flat +[release-image]: http://img.shields.io/badge/release-0.11.0-blue.svg?style=flat [releases]: https://github.com/snowplow/snowplow-mini/releases [license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat diff --git a/VERSION b/VERSION index 57121573..d9df1bbc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.10.1 +0.11.0 diff --git a/integration/integration_test.sh b/integration/integration_test.sh index 4181f31a..e251b493 100755 --- a/integration/integration_test.sh +++ b/integration/integration_test.sh @@ -1,13 +1,14 @@ #!/bin/bash -# Send good and bad events +# Send good and bad events, and test that the default redirect path is disabled COUNTER=0 while [ $COUNTER -lt 10 ]; do curl http://localhost:8080/i?e=pv curl http://localhost:8080/i + curl http://localhost:8080/r/tp2 let COUNTER=COUNTER+1 done -sleep 120 +sleep 30 # Assertions good_count="$(curl --silent -XGET 'http://localhost:9200/good/good/_count' | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["count"]')" diff --git a/provisioning/resources/configs/Caddyfile b/provisioning/resources/configs/Caddyfile index ea6f91b6..2c86b6b3 100644 --- a/provisioning/resources/configs/Caddyfile +++ b/provisioning/resources/configs/Caddyfile @@ -17,7 +17,9 @@ redir /nsqadmin /nsqadmin/ redir /elasticsearch /elasticsearch/ - proxy / localhost:8080 + proxy / localhost:8080 { + header_upstream -Authorization + } proxy /home localhost:3000 { without /home diff --git a/provisioning/resources/configs/snowplow-stream-collector.hocon b/provisioning/resources/configs/snowplow-stream-collector.hocon index 875ac751..470832c5 100644 --- a/provisioning/resources/configs/snowplow-stream-collector.hocon +++ b/provisioning/resources/configs/snowplow-stream-collector.hocon @@ -64,6 +64,8 @@ collector { body = "302, redirecting" } + enableDefaultRedirect = false + redirectMacro { enabled = false placeholder = "[TOKEN]" diff --git a/provisioning/resources/ui/index.html b/provisioning/resources/ui/index.html index ca4439af..b21b57cf 100644 --- a/provisioning/resources/ui/index.html +++ b/provisioning/resources/ui/index.html @@ -14,7 +14,7 @@

SNOWPLOW MINI

diff --git a/provisioning/resources/ui/js/components/Overview.tsx b/provisioning/resources/ui/js/components/Overview.tsx index b6cf6ffe..7209a593 100644 --- a/provisioning/resources/ui/js/components/Overview.tsx +++ b/provisioning/resources/ui/js/components/Overview.tsx @@ -42,14 +42,17 @@ export class Overview extends React.Component<{}, {}> {

3. Understanding how Snowplow Mini works

Quicklinks:

The software stack installed: