From ee1f1404269fa06c49beb537369187e5ed4a0d5c Mon Sep 17 00:00:00 2001 From: David Pilato Date: Thu, 10 Sep 2020 09:14:52 +0200 Subject: [PATCH] Update to 7.9.1 * Add support for department numbers in index names * Add Kibana saved objects (maps, index patterns, dashboards) * Fix a temporary bug with Kibana when importing Logstash pipeline (https://github.com/elastic/kibana/issues/76611) --- .env | 2 +- cloud/ingest-bano.json | 21 +++++++++++++++++++++ cloud/template-bano.json | 4 ++-- docker-compose.yml | 1 + download.sh | 6 +++++- download_region_manual.sh | 23 +++++++++++++++++++++++ filebeat-config/filebeat.yml | 2 +- kibana-config/bano.ndjson | 6 ++++++ logstash-config/pipeline/bano.conf | 9 +++++---- logstash-config/pipeline/beats.conf | 10 ++++++++-- logstash-config/pipeline/load_pipeline.sh | 2 ++ script.txt | 23 ++++++++++++++++------- setup-cloud.sh | 15 +++++++++++++++ setup.sh | 8 +++++++- 14 files changed, 113 insertions(+), 19 deletions(-) create mode 100755 download_region_manual.sh create mode 100644 kibana-config/bano.ndjson diff --git a/.env b/.env index b7e1bd8..041db40 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ -ELASTIC_VERSION=7.6.0 +ELASTIC_VERSION=7.9.1 ELASTIC_PASSWORD=changeme INJECTOR_VERSION=7.0 diff --git a/cloud/ingest-bano.json b/cloud/ingest-bano.json index 51b0236..5289677 100644 --- a/cloud/ingest-bano.json +++ b/cloud/ingest-bano.json @@ -15,6 +15,21 @@ ] } }, + { + "gsub": { + "field": "log.file.path", + "pattern": "\\/bano-data\\/bano-", + "replacement": "", + "target_field": "dept" + } + }, + { + "gsub": { + "field": "dept", + "pattern": "\\.csv", + "replacement": "" + } + }, { "convert": { "field": "location.lat", @@ -39,6 +54,12 @@ "message" ] } + }, + { + "set": { + "field": "_index", + "value": "bano-{{dept}}" + } } ] } diff --git a/cloud/template-bano.json b/cloud/template-bano.json index 14a8fcb..833438d 100644 --- a/cloud/template-bano.json +++ b/cloud/template-bano.json @@ -66,10 +66,10 @@ } } }, - "region": { + "id": { "type": "keyword" }, - "id": { + "dept": { "type": "keyword" }, "source": { diff --git a/docker-compose.yml b/docker-compose.yml index 4b878c1..0f774b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,7 @@ services: environment: - ELASTICSEARCH_USERNAME=elastic - ELASTICSEARCH_PASSWORD=$ELASTIC_PASSWORD + - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=AZERTYUIOPAZERTYUIOPAZERTYUIOP01 links: - elasticsearch ports: diff --git a/download.sh b/download.sh index 5e3d930..14892f0 100755 --- a/download.sh +++ b/download.sh @@ -5,7 +5,11 @@ echo "Download all BANO data" export SOURCE_DIR=~/Documents/Elasticsearch/Talks/postal_addresses/demo/ DEPTS=95 -for i in {1..19} $(seq 21 $DEPTS) {971..974} {976..976} ; do +for i in {1..7} {10..19} $(seq 21 $DEPTS) {971..974} {976..976} ; do DEPT=$(printf %02d $i) $SOURCE_DIR/download_region.sh $DEPT done + +# We need to manually download 08 and 09 departments as there is an issue on Linux +$SOURCE_DIR/download_region_manual.sh 08 +$SOURCE_DIR/download_region_manual.sh 09 diff --git a/download_region_manual.sh b/download_region_manual.sh new file mode 100755 index 0000000..06ba3c7 --- /dev/null +++ b/download_region_manual.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +echo "Downloading BANO region $1" + +DATASOURCE_DIR=~/Documents/Elasticsearch/Talks/postal_addresses/demo/bano-data + +import_region () { + export REGION=$1 + FILE=$DATASOURCE_DIR/bano-$REGION.csv + URL=http://bano.openstreetmap.fr/data/bano-$REGION.csv + # We import the region from openstreet map if not available yet + if [ ! -e $FILE ] ; then + echo "Fetching $FILE from $URL" + wget $URL -P $DATASOURCE_DIR + fi +} + +if [ ! -e $DATASOURCE_DIR ] ; then + echo "Creating $DATASOURCE_DIR dir" + mkdir $DATASOURCE_DIR +fi + +import_region $1 diff --git a/filebeat-config/filebeat.yml b/filebeat-config/filebeat.yml index 9ca040d..d2ff376 100644 --- a/filebeat-config/filebeat.yml +++ b/filebeat-config/filebeat.yml @@ -1,7 +1,7 @@ filebeat.inputs: - type: log paths: - - /bano-data/bano-95.csv + - /bano-data/bano-*.csv output.logstash: hosts: ["logstash:5044"] diff --git a/kibana-config/bano.ndjson b/kibana-config/bano.ndjson new file mode 100644 index 0000000..ae718b1 --- /dev/null +++ b/kibana-config/bano.ndjson @@ -0,0 +1,6 @@ +{"attributes":{"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"address.city\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"address.city.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"address.city\"}}},{\"name\":\"address.number\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"address.street_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"address.zipcode\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dept\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"id\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"location\",\"type\":\"geo_point\",\"esTypes\":[\"geo_point\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]","title":"bano-*"},"id":"3e5b6620-edf1-11ea-b21f-fd00ababe7f7","migrationVersion":{"index-pattern":"7.6.0"},"references":[],"type":"index-pattern","updated_at":"2020-09-04T09:39:03.418Z","version":"WzUwMywxXQ=="} +{"attributes":{"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"address.city\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"address.country\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"address.countrycode\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"address.location\",\"type\":\"geo_point\",\"esTypes\":[\"geo_point\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"address.number\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"address.number.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"address.number\"}}},{\"name\":\"address.street_name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"address.street_name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"address.street_name\"}}},{\"name\":\"address.zipcode\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"children\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dateofbirth\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"gender\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.cars\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.electronic\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.fashion\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.food\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.garden\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.hifi\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.music\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.shoes\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.toys\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false}]","title":"person-new"},"id":"a5695b90-ee93-11ea-95bf-9db57f641680","migrationVersion":{"index-pattern":"7.6.0"},"references":[],"type":"index-pattern","updated_at":"2020-09-04T09:47:30.222Z","version":"WzY3NywxXQ=="} +{"attributes":{"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"address.city\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"address.country\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"address.countrycode\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"address.location\",\"type\":\"geo_point\",\"esTypes\":[\"geo_point\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"address.zipcode\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"children\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dateofbirth\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"gender\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.cars\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.electronic\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.fashion\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.food\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.garden\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.hifi\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.music\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.shoes\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"marketing.toys\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false}]","title":"person"},"id":"9adba2f0-ee93-11ea-95bf-9db57f641680","migrationVersion":{"index-pattern":"7.6.0"},"references":[],"type":"index-pattern","updated_at":"2020-09-04T09:47:12.210Z","version":"WzY3NSwxXQ=="} +{"attributes":{"description":"","layerListJSON":"[{\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"isAutoSelect\":true},\"id\":\"a3997a89-dd3a-4615-81c2-95bf46d03b30\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"style\":{\"type\":\"TILE\"},\"type\":\"VECTOR_TILE\"},{\"joins\":[{\"leftField\":\"insee\",\"right\":{\"type\":\"ES_TERM_SOURCE\",\"id\":\"97b6632b-9107-4cb0-abc6-f917ea2c01b7\",\"indexPatternTitle\":\"bano-*\",\"term\":\"dept\",\"metrics\":[{\"type\":\"count\"}],\"indexPatternRefName\":\"layer_1_join_0_index_pattern\"}}],\"sourceDescriptor\":{\"type\":\"EMS_FILE\",\"id\":\"france_departments\",\"tooltipProperties\":[\"insee\"]},\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"DYNAMIC\",\"options\":{\"color\":\"Green to Red\",\"colorCategory\":\"palette_0\",\"field\":{\"name\":\"__kbnjoin__count__97b6632b-9107-4cb0-abc6-f917ea2c01b7\",\"origin\":\"join\"},\"fieldMetaOptions\":{\"isEnabled\":true,\"sigma\":3},\"type\":\"ORDINAL\",\"useCustomColorRamp\":false}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#3d3d3d\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":6}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}}},\"isTimeAware\":true},\"id\":\"a78bceeb-7fc5-4b46-9def-9c59c980157e\",\"label\":null,\"minZoom\":0,\"maxZoom\":8,\"alpha\":0.75,\"visible\":true,\"type\":\"VECTOR\"},{\"sourceDescriptor\":{\"geoField\":\"location\",\"filterByMapBounds\":true,\"scalingType\":\"CLUSTERS\",\"topHitsSize\":1,\"id\":\"d0883997-5917-4270-806d-2af520169064\",\"type\":\"ES_SEARCH\",\"tooltipProperties\":[\"address.city\",\"address.number\",\"address.street_name\",\"dept\"],\"sortField\":\"\",\"sortOrder\":\"desc\",\"indexPatternRefName\":\"layer_2_source_index_pattern\"},\"id\":\"1d65e6a4-957c-4a14-af83-057a50e51d90\",\"label\":\"Bano points\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"landmark\"}},\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#D6BF57\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":3}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":6}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}}},\"isTimeAware\":true},\"type\":\"BLENDED_VECTOR\",\"joins\":[]}]","mapStateJSON":"{\"zoom\":5,\"center\":{\"lon\":0.29615,\"lat\":46.74256},\"timeFilters\":{\"from\":\"now-15m\",\"to\":\"now\"},\"refreshConfig\":{\"isPaused\":true,\"interval\":0},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"settings\":{\"initialLocation\":\"FIXED_LOCATION\",\"fixedLocation\":{\"lat\":46.74256,\"lon\":0.29615,\"zoom\":5},\"browserLocation\":{\"zoom\":5},\"maxZoom\":18,\"minZoom\":5,\"showSpatialFilters\":true,\"spatialFiltersAlpa\":0.3,\"spatialFiltersFillColor\":\"#DA8B45\",\"spatialFiltersLineColor\":\"#DA8B45\"}}","title":"Bano dataset","uiStateJSON":"{\"isLayerTOCOpen\":false,\"openTOCDetails\":[]}"},"id":"5acb1a70-edf2-11ea-b21f-fd00ababe7f7","migrationVersion":{"map":"7.9.0"},"references":[{"id":"3e5b6620-edf1-11ea-b21f-fd00ababe7f7","name":"layer_1_join_0_index_pattern","type":"index-pattern"},{"id":"3e5b6620-edf1-11ea-b21f-fd00ababe7f7","name":"layer_2_source_index_pattern","type":"index-pattern"}],"type":"map","updated_at":"2020-09-04T09:46:29.475Z","version":"WzY3MiwxXQ=="} +{"attributes":{"description":"","layerListJSON":"[{\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"isAutoSelect\":true},\"id\":\"46f732fb-e4e0-49b7-98f4-ffe353b06b6b\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"style\":{\"type\":\"TILE\"},\"type\":\"VECTOR_TILE\"},{\"sourceDescriptor\":{\"geoField\":\"address.location\",\"filterByMapBounds\":true,\"scalingType\":\"CLUSTERS\",\"topHitsSize\":1,\"id\":\"d1d36397-a4fb-44ee-b165-f1e45966e9a7\",\"type\":\"ES_SEARCH\",\"tooltipProperties\":[\"address.city\",\"gender\",\"name\"],\"sortField\":\"\",\"sortOrder\":\"desc\",\"indexPatternRefName\":\"layer_1_source_index_pattern\"},\"id\":\"b3169562-631d-4821-ad19-1bc9385ebb7e\",\"label\":\"Persons Original Dataset\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#54B399\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#41937c\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":6}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}}},\"isTimeAware\":true},\"type\":\"BLENDED_VECTOR\",\"joins\":[],\"query\":{\"query\":\"address.city : \\\"Cergy\\\" \",\"language\":\"kuery\"}},{\"sourceDescriptor\":{\"geoField\":\"address.location\",\"filterByMapBounds\":true,\"scalingType\":\"CLUSTERS\",\"topHitsSize\":1,\"id\":\"f632a2d5-4fa2-41b7-9ec6-3e5c07659ed2\",\"type\":\"ES_SEARCH\",\"tooltipProperties\":[\"address.city\",\"address.number\",\"address.street_name\",\"gender\",\"name\"],\"sortField\":\"\",\"sortOrder\":\"desc\",\"indexPatternRefName\":\"layer_2_source_index_pattern\"},\"id\":\"7a5f5a06-d96e-409b-ab53-f8cfb0798fb2\",\"label\":\"Persons Corrected Dataset\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#6092C0\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#4379aa\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":6}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}}},\"isTimeAware\":true},\"type\":\"BLENDED_VECTOR\",\"joins\":[]}]","mapStateJSON":"{\"zoom\":12.77,\"center\":{\"lon\":2.05523,\"lat\":49.03777},\"timeFilters\":{\"from\":\"now-15m\",\"to\":\"now\"},\"refreshConfig\":{\"isPaused\":true,\"interval\":0},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"settings\":{\"initialLocation\":\"FIXED_LOCATION\",\"fixedLocation\":{\"lat\":49.03951,\"lon\":2.04695,\"zoom\":12},\"browserLocation\":{\"zoom\":10},\"maxZoom\":18,\"minZoom\":8,\"showSpatialFilters\":true,\"spatialFiltersAlpa\":0.3,\"spatialFiltersFillColor\":\"#DA8B45\",\"spatialFiltersLineColor\":\"#DA8B45\"}}","title":"Persons Dataset","uiStateJSON":"{\"isLayerTOCOpen\":true,\"openTOCDetails\":[]}"},"id":"22ae24f0-ee94-11ea-95bf-9db57f641680","migrationVersion":{"map":"7.9.0"},"references":[{"id":"9adba2f0-ee93-11ea-95bf-9db57f641680","name":"layer_1_source_index_pattern","type":"index-pattern"},{"id":"a5695b90-ee93-11ea-95bf-9db57f641680","name":"layer_2_source_index_pattern","type":"index-pattern"}],"type":"map","updated_at":"2020-09-04T09:53:06.087Z","version":"Wzc2MSwxXQ=="} +{"exportedCount":5,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/logstash-config/pipeline/bano.conf b/logstash-config/pipeline/bano.conf index 924736b..5ad35e8 100644 --- a/logstash-config/pipeline/bano.conf +++ b/logstash-config/pipeline/bano.conf @@ -10,11 +10,11 @@ input { } filter { elasticsearch { - hosts => ["http://elasticsearch:9200"] - user => "elastic" - password => "changeme" + hosts => ["https://9294b959a68c4e64b8a51f1b88105ab9.europe-west1.gcp.cloud.es.io:9243"] + user => "bano" + password => "banobano" query_template => "/usr/share/logstash/config/search-by-geo.json" - index => "bano" + index => "bano-*" fields => { "address" => "[address_new]" "location" => "[address][location]" @@ -34,6 +34,7 @@ filter { output { # stdout { codec => rubydebug } elasticsearch { + manage_template => false hosts => ["http://elasticsearch:9200"] user => "elastic" password => "changeme" diff --git a/logstash-config/pipeline/beats.conf b/logstash-config/pipeline/beats.conf index 886cae0..ecc380a 100644 --- a/logstash-config/pipeline/beats.conf +++ b/logstash-config/pipeline/beats.conf @@ -9,14 +9,19 @@ filter { columns => [ "id","number","street_name","zipcode","city","source","latitude","longitude" ] - remove_field => [ "message", "host", "@timestamp", "@version", "input", "log", "ecs", "agent", "tags" ] + remove_field => [ "message", "host", "@timestamp", "@version", "input", "ecs", "agent", "tags" ] } mutate { + gsub => [ + "[log][file][path]", "\/bano-data\/bano-", "", + "[log][file][path]", "\.csv", "" + ] convert => { "longitude" => "float" } convert => { "latitude" => "float" } } mutate { rename => { + "[log][file][path]" => "dept" "longitude" => "[location][lon]" "latitude" => "[location][lat]" "number" => "[address][number]" @@ -24,12 +29,13 @@ filter { "zipcode" => "[address][zipcode]" "city" => "[address][city]" } + remove_field => [ "log" ] } } output { elasticsearch { hosts => ["http://elasticsearch:9200"] - index => "bano" + index => "bano-%{[dept]}" user => "elastic" password => "changeme" document_id => "%{[id]}" diff --git a/logstash-config/pipeline/load_pipeline.sh b/logstash-config/pipeline/load_pipeline.sh index 89da627..d7fee17 100755 --- a/logstash-config/pipeline/load_pipeline.sh +++ b/logstash-config/pipeline/load_pipeline.sh @@ -10,6 +10,8 @@ generate_data() { cat < [ "id","number","street_name","zipcode","city","source","latitude","longitude" ] - remove_field => [ "message", "host", "@timestamp", "@version", "input", "log", "ecs", "agent", "tags" ] + remove_field => [ "message", "host", "@timestamp", "@version", "input", "ecs", "agent", "tags" ] } mutate { + gsub => [ + "[log][file][path]", "\/bano-data\/bano-", "", + "[log][file][path]", "\.csv", "" + ] convert => { "longitude" => "float" } convert => { "latitude" => "float" } } mutate { rename => { + "[log][file][path]" => "dept" "longitude" => "[location][lon]" "latitude" => "[location][lat]" "number" => "[address][number]" @@ -176,12 +184,13 @@ filter { "zipcode" => "[address][zipcode]" "city" => "[address][city]" } + remove_field => [ "log" ] } } output { elasticsearch { hosts => ["http://elasticsearch:9200"] - index => "bano" + index => "bano-%{[dept]}" user => "elastic" password => "changeme" document_id => "%{[id]}" @@ -189,11 +198,11 @@ output { } # Step 4-2: start filebeat and check how addresses are coming -GET bano/_count -GET bano/_search?size=0&track_total_hits=true +GET bano*/_count +GET bano*/_search?size=0&track_total_hits=true # Step 5-1: search for addresses -GET bano/_search?track_total_hits=true +GET bano*/_search?track_total_hits=true { "size": 1, "query": { @@ -210,7 +219,7 @@ GET bano/_search?track_total_hits=true } # Step 5-2: search by geo point -GET bano/_search?track_total_hits=true +GET bano*/_search?track_total_hits=true { "size": 1, "sort": [ @@ -226,7 +235,7 @@ GET bano/_search?track_total_hits=true } # Step 5-3: search by points with some optimization -GET bano/_search?track_total_hits=true +GET bano*/_search?track_total_hits=true { "size": 1, "query": { diff --git a/setup-cloud.sh b/setup-cloud.sh index 80270b6..a5b4114 100755 --- a/setup-cloud.sh +++ b/setup-cloud.sh @@ -14,9 +14,24 @@ done echo -ne '\n' echo Elasticsearch is now up. + +echo Elasticsearch $ELASTIC_VERSION must be available on $KIBANA_URL +echo -ne "Waiting for kibana" + +until curl -u elastic:$CLOUD_PASSWORD -s "$KIBANA_URL/app/home#/" | grep "Elastic" > /dev/null; do + sleep 1 + echo -ne '.' +done + +echo -ne '\n' +echo Kibana is now up. + echo Defining bano ingest pipeline curl -XPUT "$CLOUD_URL/_ingest/pipeline/bano" -u elastic:$CLOUD_PASSWORD -H 'Content-Type: application/json' -d'@cloud/ingest-bano.json' ; echo echo Defining bano index template curl -XPUT "$CLOUD_URL/_template/bano" -u elastic:$CLOUD_PASSWORD -H 'Content-Type: application/json' -d'@cloud/template-bano.json' ; echo +echo Installing Kibana Objects +curl -XPOST "$KIBANA_URL/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@kibana-config/bano.ndjson -u elastic:$CLOUD_PASSWORD ; echo + diff --git a/setup.sh b/setup.sh index 358ae80..08fe8fb 100755 --- a/setup.sh +++ b/setup.sh @@ -6,7 +6,7 @@ echo docker-compose down -v echo docker-compose up echo -ne "Waiting for kibana" -until curl -s "http://localhost:5601/login" | grep "Loading Kibana" > /dev/null; do +until curl -s "http://localhost:5601/login" | grep "Elastic" > /dev/null; do sleep 1 echo -ne '.' done @@ -41,6 +41,12 @@ curl -XDELETE http://localhost:9200/banotest -u elastic:$ELASTIC_PASSWORD ; echo echo Removing existing person data curl -XDELETE http://localhost:9200/person -u elastic:$ELASTIC_PASSWORD ; echo +echo Installing Kibana Objects +curl -XPOST "http://localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@kibana-config/bano.ndjson -u elastic:$ELASTIC_PASSWORD ; echo + +echo Defining bano ingest pipeline +curl -XPUT "http://localhost:9200/_ingest/pipeline/bano" -u elastic:$ELASTIC_PASSWORD -H 'Content-Type: application/json' -d'@cloud/ingest-bano.json' ; echo + echo Injecting person dataset injector/injector.sh