Introduce Quickstart deployment mode

installer/compose/README.md

@@ -48,6 +48,7 @@ We provide three options to get you going:
 - **default**: the standard installation, uses Kafka as internal message broker (recommended)
 - **nats**: the standard installation which uses Nats as message broker (recommended for new installations)
 - **full**:  contains experimental Flink wrappers
+- **quickstart**:  contains predefined assets
 
 The ``nats`` version will become the default version in a later release. You can already try it for new installations, 
 but there's not yet an automatic migration from current Kafka-based installations to Nats.
@@ -90,6 +91,18 @@ docker-compose -f docker-compose.full.yml down
 #docker-compose -f docker-compose.full.yml down
 ```
 
+We introduce quickstart deployment that comes with a set of predefined StreamPipes assets, to use the quickstart mode, just build the Docker image and start **quickstart** option:
+```bash
+docker-compose -f docker-compose.quickstart.yml build script-runner
+docker-compose -f docker-compose.quickstart.yml up -d
+# go to after all services are started http://localhost
+```
+Stopping the **quickstart** option:
+```bash
+docker-compose -f docker-compose.quickstart.yml down
+#docker-compose -f docker-compose.quickstart.yml down
+```
+
 ## Update services
 To actively pull the latest available Docker images use:
 ```bash

installer/compose/docker-compose.quickstart.yml

@@ -0,0 +1,195 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+version: "3.4"
+
+# global logging
+x-logging:
+  &default-logging
+  options:
+    max-size: '12m'
+    max-file: '5'
+  driver: json-file
+
+services:
+
+  # StreamPipes services
+  backend:
+    image: "${SP_DOCKER_REGISTRY}/backend:${SP_VERSION}"
+    depends_on:
+      - consul
+      - couchdb
+    volumes:
+      - backend:/root/.streampipes
+      - files:/spImages
+    logging: *default-logging
+    restart: unless-stopped
+    networks:
+      spnet:
+
+  ui:
+    image: "${SP_DOCKER_REGISTRY}/ui:${SP_VERSION}"
+    ports:
+      - "80:8088"
+    depends_on:
+      - couchdb
+      - consul
+      - backend
+    volumes:
+      - nginx:/etc/nginx/
+    logging: *default-logging
+    restart: unless-stopped
+    networks:
+      spnet:
+
+  consul:
+    image: consul:1.14.3
+    environment:
+      - "CONSUL_LOCAL_CONFIG={\"disable_update_check\": true}"
+      - "CONSUL_BIND_INTERFACE=eth0"
+      - "CONSUL_HTTP_ADDR=0.0.0.0"
+    entrypoint:
+      - consul
+      - agent
+      - -server
+      - -bootstrap-expect=1
+      - -data-dir=/consul/data
+      - -node=consul-one
+      - -bind={{ GetInterfaceIP "eth0" }}
+      - -client=0.0.0.0
+      - -enable-script-checks=true
+      - -ui
+    volumes:
+      - consul:/consul/data
+    logging: *default-logging
+    restart: unless-stopped
+    networks:
+      spnet:
+        ipv4_address: ${SP_CONSUL_CONTAINER_IP}
+
+  couchdb:
+    image: couchdb:3.3.1
+    environment:
+      - COUCHDB_USER=admin
+      - COUCHDB_PASSWORD=admin
+    volumes:
+      - couchdb:/opt/couchdb/data
+    logging: *default-logging
+    restart: unless-stopped
+    networks:
+      spnet:
+
+  kafka:
+    image: fogsyio/kafka:2.2.0
+    hostname: kafka
+    depends_on:
+      - zookeeper
+    environment:
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092
+      KAFKA_LISTENERS: PLAINTEXT://:9092
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_ADVERTISED_HOST_NAME: kafka
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_MESSAGE_MAX_BYTES: 5000012
+      KAFKA_FETCH_MESSAGE_MAX_BYTES: 5000012
+      KAFKA_REPLICA_FETCH_MAX_BYTES: 10000000
+    volumes:
+      - kafka:/kafka
+      - /var/run/docker.sock:/var/run/docker.sock
+    logging: *default-logging
+    restart: unless-stopped
+    networks:
+      spnet:
+
+  zookeeper:
+    image: fogsyio/zookeeper:3.4.13
+    volumes:
+      - zookeeper:/opt/zookeeper-3.4.13
+    logging: *default-logging
+    restart: unless-stopped
+    networks:
+      spnet:
+
+  influxdb:
+    image: influxdb:2.6
+    environment:
+      - INFLUXDB_DATA_ENGINE=tsm1
+      - INFLUXDB_REPORTING_DISABLED=false
+      - INFLUXDB_ADMIN_ENABLED=true
+      - DOCKER_INFLUXDB_INIT_USERNAME=admin
+      - DOCKER_INFLUXDB_INIT_PASSWORD=sp-admin
+      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=sp-admin
+      - DOCKER_INFLUXDB_INIT_ORG=sp
+      - DOCKER_INFLUXDB_INIT_BUCKET=sp
+      - DOCKER_INFLUXDB_INIT_MODE=${SP_INFLUX_INIT_MODE}
+    volumes:
+      - influxdb:/var/lib/influxdb
+      - influxdb2:/var/lib/influxdb2
+    logging: *default-logging
+    restart: unless-stopped
+    networks:
+      spnet:
+
+  # Lite version pipeline elements
+  extensions-all-jvm:
+    image: "${SP_DOCKER_REGISTRY}/extensions-all-jvm:${SP_VERSION}"
+    depends_on:
+      - consul
+    volumes:
+      - files:/spImages
+    logging: *default-logging
+    restart: unless-stopped
+    networks:
+      spnet:
+
+  sources-watertank-simulator:
+    image: "${SP_DOCKER_REGISTRY}/sources-watertank-simulator:${SP_VERSION}"
+    depends_on:
+      - consul
+      - kafka
+    logging: *default-logging
+    restart: unless-stopped
+    networks:
+      spnet:
+  script-runner:
+    build: ./quickstart
+    depends_on:
+      - backend
+    volumes:
+      - ./quickstart/zip_folder:/zip_folder
+    networks:
+      spnet:
+
+
+
+volumes:
+  backend:
+  consul:
+  connect:
+  couchdb:
+  kafka:
+  zookeeper:
+  influxdb:
+  influxdb2:
+  files:
+  nginx:
+
+
+networks:
+  spnet:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: ${SP_SUBNET}

installer/compose/quickstart/Dockerfile

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Use an official base image
+FROM ubuntu:latest
+
+# Install curl and jq
+RUN apt-get update \
+    && apt-get install -y curl jq
+
+
+RUN curl -L -o /usr/local/bin/wait-for-it.sh https://raw.githubusercontent.com/vishnubob/wait-for-it/81b1373f17855a4dc21156cfe1694c31d7d1792e/wait-for-it.sh \
+    && chmod +x /usr/local/bin/wait-for-it.sh
+
+
+COPY upload.sh /usr/local/bin/upload.sh
+RUN chmod +x /usr/local/bin/upload.sh
+
+
+ENV WAITFORIT_TIMEOUT 60
+
+# Run the script
+CMD ["/usr/local/bin/wait-for-it.sh", "backend:8030", "--", "/usr/local/bin/upload.sh"]

installer/compose/quickstart/upload.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ -z "$SP_INITIAL_ADMIN_EMAIL" ]; then
+  USERNAME="[email protected]"
+else
+  USERNAME="$SP_INITIAL_ADMIN_EMAIL"
+fi
+
+if [ -z "$SP_INITIAL_ADMIN_PASSWORD" ]; then
+  PASSWORD="admin"
+else
+  PASSWORD="$SP_INITIAL_ADMIN_PASSWORD"
+fi
+
+JSON_TOKEN_RESPONSE=$(curl -s -X POST "http://backend:8030/streampipes-backend/api/v2/auth/login" \
+    -H "Content-Type: application/json" \
+    -d "{\"username\":\"$USERNAME\",\"password\":\"$PASSWORD\"}")
+
+TOKEN=$(echo "$JSON_TOKEN_RESPONSE" | jq -r '.accessToken')
+RESPONSE_TOKEN="Bearer $TOKEN"
+
+JSON_RESOURCE_ITEMS=$(curl -s -X GET "http://backend:8030/streampipes-backend/api/v2/rdfendpoints/items" \
+    -H "Content-Type: application/json" \
+    -H "Authorization: $RESPONSE_TOKEN")
+
+ARRAY_LENGTH=$(echo $JSON_RESOURCE_ITEMS | jq '. | length')
+
+while [ $ARRAY_LENGTH -le 100 ]; do
+    echo "StreamPipes Extensions Service not ready, waiting for 3 seconds..."
+    sleep 3
+    JSON_RESOURCE_ITEMS=$(curl -s -X GET "http://backend:8030/streampipes-backend/api/v2/rdfendpoints/items" \
+        -H "Content-Type: application/json" \
+        -H "Authorization: $RESPONSE_TOKEN")
+    ARRAY_LENGTH=$(echo $JSON_RESOURCE_ITEMS | jq '. | length')
+done
+
+ITEM_MAP_NAME_URI=$(echo $JSON_RESOURCE_ITEMS | jq -r 'map({(.name): .uri}) | add')
+
+ITEM_MAP_NAME_URI=$(echo $JSON_RESOURCE_ITEMS | jq -r 'map({(.name): .uri}) | add')
+
+for ZIP_FILE in /zip_folder/*.zip; do
+  JSON_RESPONSE=$(curl --compressed -X POST "http://backend:8030/streampipes-backend/api/v2/import/preview" \
+      -H "Authorization: $RESPONSE_TOKEN" \
+      -F "file_upload=@$ZIP_FILE")
+
+  PIPELINE_IDS=($(echo "$JSON_RESPONSE" | jq -r '.pipelines[].resourceId'))
+
+  for id in "${PIPELINE_IDS[@]}"; do
+
+    ITEMS_PIPELINES=$(curl -s -X GET "http://backend:8030/streampipes-backend/api/v2/pipelines/$id" \
+        -H "Content-Type: application/json" \
+        -H "Authorization: $RESPONSE_TOKEN")
+
+    KEYS=$(echo $ITEMS_PIPELINES | jq -r 'keys[]')
+
+    for key in $KEYS; do
+      if [ "$(echo $ITEMS_PIPELINES | jq ".${key} | type")" == '"array"' ]; then
+        key=$(echo $ITEMS_PIPELINES | jq -r ".${key}[]? | .name")
+        value=$(echo $ITEM_MAP_NAME_URI | jq -r ".\"$key\"")
+
+        curl -s -X POST "http://backend:8030/streampipes-backend/api/v2/element" \
+            -H "Content-Type: application/x-www-form-urlencoded;charset=UTF-8" \
+            -H "Authorization: $RESPONSE_TOKEN" \
+            --data-urlencode "uri=$value" \
+            --data-urlencode "publicElement=true"
+      fi
+    done
+  done
+
+  JSON_PAYLOAD="$JSON_RESPONSE"
+
+  curl -i -X POST "http://backend:8030/streampipes-backend/api/v2/import" \
+      -H "Authorization: $RESPONSE_TOKEN" \
+      -F "file_upload=@$ZIP_FILE" \
+      -F "configuration=@-;type=application/json" <<< "$JSON_PAYLOAD"
+done
+