diff --git a/.gitignore b/.gitignore index 0ce436e0b..c28faba1a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ .gradle .local-* build - +.DS_Store .project .classpath .settings @@ -10,5 +10,5 @@ bin stack-docs.iml stack-docs.ipr stack-docs.iws - +/html_docs .vscode diff --git a/docs/60_upgrade_guide.html b/docs/60_upgrade_guide.html index 5898eacea..fa81207a9 100644 --- a/docs/60_upgrade_guide.html +++ b/docs/60_upgrade_guide.html @@ -861,7 +861,7 @@ [[Upgrade Elasticsearch for Hadoop to 5.6.|https://www.elastic.co/guide/en/elasticsearch/hadoop/5.6/install.html]]\ <<endif>>\!!!Upgrade to 6.3 <<if $xpack_ml>>\ -[[Stop any running Machine Learning jobs.|https://www.elastic.co/guide/en/elastic-stack-overview/6.3/stopping-ml.html]] +[[Stop any running Machine Learning jobs.|https://www.elastic.co/guide/en/machine-learning/6.3/stopping-ml.html]] <<endif>>\ <<if $hadoop>>\ <span class="pseudo-link">Disable ingestion from Hadoop while you upgrade the stack.</span> diff --git a/docs/en/endpoint/images/help-button.png b/docs/en/endpoint/images/help-button.png new file mode 100644 index 000000000..a75325d87 Binary files /dev/null and b/docs/en/endpoint/images/help-button.png differ diff --git a/docs/en/endpoint/index.asciidoc b/docs/en/endpoint/index.asciidoc new file mode 100644 index 000000000..70ed8a9fc --- /dev/null +++ b/docs/en/endpoint/index.asciidoc @@ -0,0 +1,6 @@ +[[elastic-endpoint]] += Elastic Endpoint Security + +The documentation for Elastic Endpoint Security is accessible via the management console. Log in with your credentials, then click the Help button in the upper-right corner to view the Knowledge Base. + +image::images/help-button.png[Help button] diff --git a/docs/en/getting-started/docker/.env b/docs/en/getting-started/docker/.env new file mode 100644 index 000000000..2d2edc9db --- /dev/null +++ b/docs/en/getting-started/docker/.env @@ -0,0 +1,3 @@ +COMPOSE_PROJECT_NAME=es +CERTS_DIR=/usr/share/elasticsearch/config/certificates +VERSION={version} diff --git a/docs/en/getting-started/docker/create-certs.yml b/docs/en/getting-started/docker/create-certs.yml new file mode 100644 index 000000000..36c1573aa --- /dev/null +++ b/docs/en/getting-started/docker/create-certs.yml @@ -0,0 +1,29 @@ +version: '2.2' + +services: + create_certs: + image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION} + container_name: create_certs + command: > + bash -c ' + yum install -y -q -e 0 unzip; + if [[ ! -f /certs/bundle.zip ]]; then + bin/elasticsearch-certutil cert --silent --pem --in config/certificates/instances.yml -out /certs/bundle.zip; + unzip /certs/bundle.zip -d /certs; + fi; + chown -R 1000:0 /certs + ' + working_dir: /usr/share/elasticsearch + volumes: + - certs:/certs + - .:/usr/share/elasticsearch/config/certificates + networks: + - elastic + +volumes: + certs: + driver: local + +networks: + elastic: + driver: bridge diff --git a/docs/en/getting-started/docker/docker-compose.yml b/docs/en/getting-started/docker/docker-compose.yml new file mode 100644 index 000000000..b05040d6f --- /dev/null +++ b/docs/en/getting-started/docker/docker-compose.yml @@ -0,0 +1,87 @@ +version: '2.2' +services: + es01: + image: docker.elastic.co/elasticsearch/elasticsearch:{version} + container_name: es01 + environment: + - node.name=es01 + - cluster.name=es-docker-cluster + - discovery.seed_hosts=es02,es03 + - cluster.initial_master_nodes=es01,es02,es03 + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - data01:/usr/share/elasticsearch/data + ports: + - 9200:9200 + networks: + - elastic + + es02: + image: docker.elastic.co/elasticsearch/elasticsearch:{version} + container_name: es02 + environment: + - node.name=es02 + - cluster.name=es-docker-cluster + - discovery.seed_hosts=es01,es03 + - cluster.initial_master_nodes=es01,es02,es03 + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - data02:/usr/share/elasticsearch/data + ports: + - 9201:9201 + networks: + - elastic + + es03: + image: docker.elastic.co/elasticsearch/elasticsearch:{version} + container_name: es03 + environment: + - node.name=es03 + - cluster.name=es-docker-cluster + - discovery.seed_hosts=es01,es02 + - cluster.initial_master_nodes=es01,es02,es03 + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - data03:/usr/share/elasticsearch/data + ports: + - 9202:9202 + networks: + - elastic + + kib01: + image: docker.elastic.co/kibana/kibana:{version} + container_name: kib01 + ports: + - 5601:5601 + environment: + ELASTICSEARCH_URL: http://es01:9200 + ELASTICSEARCH_HOSTS: http://es01:9200 + networks: + - elastic + +volumes: + data01: + driver: local + data02: + driver: local + data03: + driver: local + +networks: + elastic: + driver: bridge diff --git a/docs/en/getting-started/docker/elastic-docker-tls.yml b/docs/en/getting-started/docker/elastic-docker-tls.yml new file mode 100644 index 000000000..3d975b5aa --- /dev/null +++ b/docs/en/getting-started/docker/elastic-docker-tls.yml @@ -0,0 +1,136 @@ +version: '2.2' + +services: + es01: + image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION} + container_name: es01 + environment: + - node.name=es01 + - cluster.name=es-docker-cluster + - discovery.seed_hosts=es02,es03 + - cluster.initial_master_nodes=es01,es02,es03 + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - xpack.license.self_generated.type=trial # <1> + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true # <2> + - xpack.security.http.ssl.key=$CERTS_DIR/es01/es01.key + - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt + - xpack.security.http.ssl.certificate=$CERTS_DIR/es01/es01.crt + - xpack.security.transport.ssl.enabled=true # <3> + - xpack.security.transport.ssl.verification_mode=certificate # <4> + - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt + - xpack.security.transport.ssl.certificate=$CERTS_DIR/es01/es01.crt + - xpack.security.transport.ssl.key=$CERTS_DIR/es01/es01.key + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - data01:/usr/share/elasticsearch/data + - certs:$CERTS_DIR + ports: + - 9200:9200 + networks: + - elastic + + healthcheck: + test: curl --cacert $CERTS_DIR/ca/ca.crt -s https://localhost:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi + interval: 30s + timeout: 10s + retries: 5 + + es02: + image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION} + container_name: es02 + environment: + - node.name=es02 + - cluster.name=es-docker-cluster + - discovery.seed_hosts=es01,es03 + - cluster.initial_master_nodes=es01,es02,es03 + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - xpack.license.self_generated.type=trial + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=$CERTS_DIR/es02/es02.key + - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt + - xpack.security.http.ssl.certificate=$CERTS_DIR/es02/es02.crt + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt + - xpack.security.transport.ssl.certificate=$CERTS_DIR/es02/es02.crt + - xpack.security.transport.ssl.key=$CERTS_DIR/es02/es02.key + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - data02:/usr/share/elasticsearch/data + - certs:$CERTS_DIR + networks: + - elastic + + es03: + image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION} + container_name: es03 + environment: + - node.name=es03 + - cluster.name=es-docker-cluster + - discovery.seed_hosts=es01,es02 + - cluster.initial_master_nodes=es01,es02,es03 + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - xpack.license.self_generated.type=trial + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=$CERTS_DIR/es03/es03.key + - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt + - xpack.security.http.ssl.certificate=$CERTS_DIR/es02/es02.crt + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt + - xpack.security.transport.ssl.certificate=$CERTS_DIR/es03/es03.crt + - xpack.security.transport.ssl.key=$CERTS_DIR/es03/es03.key + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - data03:/usr/share/elasticsearch/data + - certs:$CERTS_DIR + networks: + - elastic + kib01: + image: docker.elastic.co/kibana/kibana:${VERSION} + container_name: kib01 + depends_on: {"es01": {"condition": "service_healthy"}} + ports: + - 5601:5601 + environment: + SERVERNAME: localhost + ELASTICSEARCH_URL: https://es01:9200 + ELASTICSEARCH_HOSTS: https://es01:9200 + ELASTICSEARCH_USERNAME: kibana + ELASTICSEARCH_PASSWORD: CHANGEME + ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: $CERTS_DIR/ca/ca.crt + SERVER_SSL_ENABLED: "true" + SERVER_SSL_KEY: $CERTS_DIR/kib01/kib01.key + SERVER_SSL_CERTIFICATE: $CERTS_DIR/kib01/kib01.crt + volumes: + - certs:$CERTS_DIR + networks: + - elastic +volumes: + data01: + driver: local + data02: + driver: local + data03: + driver: local + certs: + driver: local + +networks: + elastic: + driver: bridge diff --git a/docs/en/getting-started/docker/instances.yml b/docs/en/getting-started/docker/instances.yml new file mode 100644 index 000000000..affbb76fd --- /dev/null +++ b/docs/en/getting-started/docker/instances.yml @@ -0,0 +1,26 @@ +instances: + - name: es01 + dns: + - es01 + - localhost + ip: + - 127.0.0.1 + + - name: es02 + dns: + - es02 + - localhost + ip: + - 127.0.0.1 + + - name: es03 + dns: + - es03 + - localhost + ip: + - 127.0.0.1 + + - name: 'kib01' + dns: + - kib01 + - localhost \ No newline at end of file diff --git a/docs/en/getting-started/get-started-docker.asciidoc b/docs/en/getting-started/get-started-docker.asciidoc index 173e845bd..f162e61c1 100644 --- a/docs/en/getting-started/get-started-docker.asciidoc +++ b/docs/en/getting-started/get-started-docker.asciidoc @@ -1,12 +1,262 @@ [[get-started-docker]] == Running the {stack} on Docker -The Elastic Docker registry contains Docker images for all the products in the -{stack}. For more information, see: +The Elastic Docker registry contains Docker images for all the products in +the {stack}: https://www.docker.elastic.co/. + +[float] +[[run-stack-docker]] +=== Run with Docker Compose + +To get the default distributions of {es} and {kib} up and running in Docker, +you can use Docker Compose. + +. Create a `docker-compose.yml` file for the Elastic Stack. +The following example brings up a three node cluster and Kibana so you can see how things work. +This all-in-one configuration is a handy way to bring up your first dev cluster before +you build a distributed deployment with multiple hosts. ++ +ifeval::["{release-state}"=="unreleased"] +NOTE: Version {version} of {es} has not been released, +so the sample compose file is not yet available for this version. +See the {stack-gs-current}/get-started-docker.html[current version] for the latest sample files. +endif::[] +ifeval::["{release-state}"!="unreleased"] +["source","yaml",subs="attributes"] +-------------------------------------------- +include::docker/docker-compose.yml[] +-------------------------------------------- +endif::[] +// tag::docker-memory[] +. Make sure Docker Engine is allotted at least 4GiB of memory. +In Docker Desktop, you configure resource usage on the Advanced tab in Preference (macOS) +or Settings (Windows). +// end::docker-memory[] + +. Run `docker-compose` to bring up the three-node {es} cluster and {kib}: ++ +["source","sh",subs="attributes"] +-------------------------------------------- +docker-compose up +-------------------------------------------- + +. Submit a `_cat/nodes` request to see that the nodes are up and running: ++ +[source,sh] +-------------------------------------------------- +curl -X GET "localhost:9200/_cat/nodes?v&pretty" +-------------------------------------------------- + +. Open {kib} to load sample data and interact with the cluster: +http://localhost:5601. + +When you're done experimenting, you can tear down the containers and +volumes by running `docker-compose down -v`. + +[float] +[[get-started-docker-tls]] +=== Run in Docker with TLS enabled + +When security is enabled with a https://www.elastic.co/subscriptions[Gold or Platinum license], +Transport Layer Security (TLS) encryption must be configured for the {es} transport layer. +While it is possible to use a trial license without setting up TLS, +we advise securing your stack from the start. + +To get an {es} cluster and {kib} up and running in Docker with security enabled, +you can use Docker Compose: + +. Create the following compose and configuration files. +These files are also available from the +https://github.com/elastic/stack-docs/blob/master/docs/en/getting-started/docker/[elastic/stack-docs] +repository on GitHub. ++ +-- +ifeval::["{release-state}"=="unreleased"] +NOTE: Version {version} of {es} has not been released, +so the sample compose and configuration files are not yet available for this version. +See the {stack-gs-current}/get-started-docker.html[current version] for the latest sample files. +endif::[] + +* `instances.yml` identifies the instances you need to create certificates for. +* `.env` sets environment variables to specify the {es} version and +the location where the {es} certificates will be created. +* `create-certs.yml` is a Docker Compose file that launches a container to generate the certificates +for {es} and {kib}. +* `elastic-docker-tls.yml` is a Docker Compose file that brings up a three-node {es} cluster +and a {kib} instance with TLS enabled so you can see how things work. +This all-in-one configuration is a handy way to bring up your first dev cluster before +you build a distributed deployment with multiple hosts. + +ifeval::["{release-state}"!="unreleased"] +.`instances.yml`: +["source","yaml"] +---- +include::docker/instances.yml[] +---- + +.`.env`: +["source","txt",subs="attributes"] +---- +include::docker/.env[] +---- + +.`create-certs.yml`: +["source","txt"] +---- +include::docker/create-certs.yml[] +---- + +.`elastic-docker-tls.yml`: +["source","txt"] +---- +include::docker/elastic-docker-tls.yml[] +---- + +<1> Generate and apply a trial license that supports Transport Layer Security. +<2> Enable Transport Layer Security to encrypt client communications. +<3> Enable Transport Layer Security to encrypt internode communications. +<4> Allow the use of self-signed certificates by not requiring hostname verification. +endif::[] +-- + +include::get-started-docker.asciidoc[tag=docker-memory] + + +. Generate certificates for {es} by bringing up the `create-certs` container: ++ +-- +["source","sh"] +---- +docker-compose -f create-certs.yml run --rm create_certs +---- + +-- + +. Bring up the three-node {es} cluster: ++ +-- +["source","sh"] +---- +docker-compose -f elastic-docker-tls.yml up -d +---- + +IMPORTANT: At this point, {kib} cannot connect to the {es} cluster. +You must generate a password for the built-in `kibana_system` user, update the `ELASTICSEARCH_PASSWORD` +in the compose file, and restart to enable {kib} to communicate with the secured cluster. + +-- + +. Run the `elasticsearch-setup-passwords` tool to generate passwords for all built-in users, +including the `kibana_system` user. If you don't use PowerShell on Windows, remove the trailing `\`characters +and join the lines before running this command. ++ +-- +["source","sh"] +---- +docker exec es01 /bin/bash -c "bin/elasticsearch-setup-passwords \ +auto --batch \ +-Expack.security.http.ssl.certificate=certificates/es01/es01.crt \ +-Expack.security.http.ssl.certificate_authorities=certificates/ca/ca.crt \ +-Expack.security.http.ssl.key=certificates/es01/es01.key \ +--url https://es01:9200" +---- + +IMPORTANT: Make a note of the generated passwords. +You must configure the `kibana_system` user password in the compose file to enable {kib} to connect to {es}, +and you'll need the password for the `elastic` superuser to +log in to {kib} and submit requests to {es}. +-- + +. Set `ELASTICSEARCH_PASSWORD` in the `elastic-docker-tls.yml` compose file to the password +generated for the `kibana_system` user. ++ +-- +ifeval::["{release-state}"=="unreleased"] +NOTE: Version {version} of {es} has not been released, +so the sample compose file is not yet available for this version. +See the {stack-gs-current}/get-started-docker.html[current version] for the latest sample files. +endif::[] + +ifeval::["{release-state}"!="unreleased"] +["source","yaml",subs=+quotes] +---- + kib01: + image: docker.elastic.co/kibana/kibana:${VERSION} + container_name: kib01 + depends_on: {"es01": {"condition": "service_healthy"}} + ports: + - 5601:5601 + environment: + SERVERNAME: localhost + ELASTICSEARCH_URL: https://es01:9200 + ELASTICSEARCH_HOSTS: https://es01:9200 + ELASTICSEARCH_USERNAME: kibana_system + **ELASTICSEARCH_PASSWORD: CHANGEME** + ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: $CERTS_DIR/ca/ca.crt + SERVER_SSL_ENABLED: "true" + SERVER_SSL_KEY: $CERTS_DIR/kib01/kib01.key + SERVER_SSL_CERTIFICATE: $CERTS_DIR/kib01/kib01.crt + volumes: + - certs:$CERTS_DIR + networks: + - elastic +---- +endif::[] +-- + +. Use `docker-compose` to restart the cluster and {kib}: ++ +-- +["source","sh"] +---- +docker-compose stop +docker-compose -f elastic-docker-tls.yml up -d +---- +-- + +. Open {kib} to load sample data and interact with the cluster: +https://localhost:5601. ++ +NOTE: Because SSL is also enabled for communications between {kib} and client browsers, +you must access {kib} via the HTTPS protocol. + +When you're done experimenting, you can tear down the containers, network, and +volumes by running `docker-compose -f elastic-docker-tls.yml down -v`. + +[float] +[[load-settings-file]] +==== Loading settings from a file + +Specifying settings for {es} and {{kib}} directly in the compose file is a convenient way to get started, +but loading settings from a file is preferable once you get past the experimental stage. + +For example, to use `es01.yml` as the configuration file for the `es01` {es} node, +you can create a bind mount in the volumes section. + +["source","yaml"] +---- + volumes: + - data01:/usr/share/elasticsearch/data + - certs:$CERTS_DIR + - ./es01.yml:/usr/share/elasticsearch/config/elasticsearch.yml +---- + +Similarly, to load {kib} settings from a file, you overwrite `/usr/share/kibana/config/kibana.yml`: + +["source","yaml"] +---- + volumes: + - certs:$CERTS_DIR + - ./kibana.yml:/usr/share/kibana/config/kibana.yml +---- + +[float] +=== Product-specific instructions for Docker + +See the product-specific documentation for information about running a specific Elastic product in Docker: -* Official Elastic Docker images: https://www.docker.elastic.co/ * {ref}/docker.html[Install {es} with Docker] -* {apm-server-ref}/running-on-docker.html[Running APM Server on Docker] +* {apm-server-ref-70}/running-on-docker.html[Running APM Server on Docker] * {auditbeat-ref}/running-on-docker.html[Running {auditbeat} on Docker] * {filebeat-ref}/running-on-docker.html[Running {filebeat} on Docker] * {heartbeat-ref}/running-on-docker.html[Running {heartbeat} on Docker] @@ -14,6 +264,3 @@ The Elastic Docker registry contains Docker images for all the products in the * {logstash-ref}/docker.html[Running {ls} on Docker] * {metricbeat-ref}/running-on-docker.html[Running {metricbeat} on Docker] * {packetbeat-ref}/running-on-docker.html[Running {packetbeat} on Docker] - - - diff --git a/docs/en/getting-started/get-started-stack.asciidoc b/docs/en/getting-started/get-started-stack.asciidoc index ff6e8c3f5..0e495081f 100644 --- a/docs/en/getting-started/get-started-stack.asciidoc +++ b/docs/en/getting-started/get-started-stack.asciidoc @@ -1,28 +1,20 @@ [[get-started-elastic-stack]] == Getting started with the {stack} -Looking for an {stack} ("ELK" tutorial) that shows how to set up the {stack}? In -this tutorial, you learn how to get up and running quickly. First you install -the core open source products: +Looking for an {stack} ("ELK") guide that shows how to quickly install and configure the {stack}? You're in the right place! You can install the {stack} on a single VM, or even on your laptop: * <> * <> * <> -* <> +* <> -Then you learn how to implement a system monitoring solution that uses -{metricbeat} to collect server metrics and ship the data to {es}, where you can -search and visualize the data by using {kib}. After you get the basic setup -working, you add {ls} for additional parsing. +After completing the installation process, learn how to implement a system monitoring solution that uses +{metricbeat} to collect server metrics and ship the data to {es}, search and visualize the data by using {kib}, and incorporate {ls} for additional parsing. -To get started, you can install the {stack} on a single VM or even on your -laptop. - -IMPORTANT: Implementing security is a critical step in setting up the {stack}. -To get up and running quickly with a sample installation, you skip those steps -right now. Before sending sensitive data across the network, make sure you -{stack-ov}/elasticsearch-security.html[secure the {stack}] and enable -{stack-ov}/encrypting-communications.html[encrypted communications]. +IMPORTANT: Implementing security is a critical step in configuring the {stack}. +This guide skips security configuration to quickly install a sample installation. Before sending sensitive data across the network, +{ref}/elasticsearch-security.html[secure the {stack}] and enable +{ref}/encrypting-communications.html[encrypted communications]. [float] [[install-prereqs]] @@ -50,13 +42,14 @@ data, such as logs or decoded network packets. You can run Elasticsearch on your own hardware, or use our https://www.elastic.co/cloud/elasticsearch-service[hosted Elasticsearch Service] on Elastic Cloud. The Elasticsearch Service is available on both AWS and GCP. -https://www.elastic.co/cloud/elasticsearch-service/signup[Try out the +{ess-trial}[Try out the Elasticsearch Service for free]. ========== To download and install {es}, open a terminal window and use the commands that work with your system (<> for Debian/Ubuntu, <> for -Redhat/Centos/Fedora, <> for OS X, and <> for Windows): +Redhat/Centos/Fedora, <> or <> for OS X, <> +for Linux, and <> for Windows): [[deb]]*deb:* @@ -117,6 +110,24 @@ cd elasticsearch-{elasticsearch_version} endif::[] +[[brew]]*brew:* + +ifeval::["{release-state}"=="unreleased"] + +Version {version} of {es} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes,callouts"] +---------------------------------------------------------------------- +brew tap elastic/tap +brew install elastic/tap/elasticsearch-full +elasticsearch +---------------------------------------------------------------------- + +endif::[] [[linux]]*linux:* @@ -174,6 +185,13 @@ endif::[] For other operating systems, go to the https://www.elastic.co/downloads/elasticsearch[{es} download] page. +TIP: The default {ref}/cluster.name.html[cluster.name] and +{ref}/node.name.html[node.name] are `elasticsearch` and your hostname, +respectively. If you plan to keep using this cluster or add more nodes, it is a +good idea to change these default values to unique names. For details about +changing these and other settings in the `elasticsearch.yml` file, see +{ref}/settings.html[Configuring {es}]. + To learn more about installing, configuring, and running {es}, read the https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html[{es} Reference]. @@ -227,8 +245,9 @@ and maps. [TIP] ========== -If you are running our hosted Elasticsearch Service on https://www.elastic.co/cloud[Elastic Cloud], -then {cloud}/ec-enable-kibana.html[Kibana can be enabled] with the flick of a switch. +Running our hosted Elasticsearch Service on https://www.elastic.co/cloud[Elastic Cloud]? +Kibana is enabled automatically in most templates, +or manually with the {cloud}/ec-enable-kibana.html[flick of a switch]. ========== We recommend that you install {kib} on the same server as {es}, @@ -239,7 +258,7 @@ file, `kibana.yml`, before starting {kib}. To download and install {kib}, open a terminal window and use the commands that work with your system: -*deb or rpm:* +*deb, rpm, or linux:* ifeval::["{release-state}"=="unreleased"] @@ -279,6 +298,25 @@ cd kibana-{kibana_version}-darwin-x86_64/ endif::[] +*brew:* + +ifeval::["{release-state}"=="unreleased"] + +Version {version} of {kib} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes"] +---------------------------------------------------------------------- +brew tap elastic/tap +brew install elastic/tap/kibana-full +kibana +---------------------------------------------------------------------- + +endif::[] + *win:* ifeval::["{release-state}"=="unreleased"] @@ -332,7 +370,7 @@ The {beats} are open source data shippers that you install as agents on your servers to send operational data to {es}. {beats} can send data directly to {es} or via {ls}, where you can further process and enhance the data. -Each Beat is a separately installable product. In this tutorial, you learn how +Each Beat is a separately installable product. In this guide, you learn how to install and run {metricbeat} with the `system` module enabled to collect system metrics. @@ -414,6 +452,42 @@ tar xzvf metricbeat-{version}-darwin-x86_64.tar.gz endif::[] +*brew:* + +ifeval::["{release-state}"=="unreleased"] + +Version {version} of {metricbeat} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes"] +---------------------------------------------------------------------- +brew tap elastic/tap +brew install elastic/tap/metricbeat-full +---------------------------------------------------------------------- + +endif::[] + +*linux:* + +ifeval::["{release-state}"=="unreleased"] + +Version {version} of {metricbeat} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes,callouts"] +------------------------------------------------ +curl -L -O https://artifacts.elastic.co/downloads/beats/metricbeat/metricbeat-{version}-linux-x86_64.tar.gz +tar xzvf metricbeat-{version}-linux-x86_64.tar.gz +------------------------------------------------ + +endif::[] + *win:* ifeval::["{release-state}"=="unreleased"] @@ -480,13 +554,20 @@ To set up the `system` module and start collecting system metrics: sudo metricbeat modules enable system ---- + -*mac:* +*mac and linux:* + [source,yaml] ---- ./metricbeat modules enable system ---- + +*brew:* ++ +[source,yaml] +---- +metricbeat modules enable system +---- ++ *win:* + [source,yaml] @@ -503,13 +584,20 @@ PS C:\Program Files\Metricbeat> .\metricbeat.exe modules enable system sudo metricbeat setup -e ---- + -*mac:* +*mac and linux:* + [source,yaml] ---- ./metricbeat setup -e ---- + +*brew:* ++ +[source,yaml] +---- +metricbeat setup -e +---- ++ *win:* + [source,yaml] @@ -521,8 +609,7 @@ The `setup` command loads the {kib} dashboards. If the dashboards are already set up, omit this command. The `-e` flag is optional and sends output to standard error instead of syslog. -[[gs-start-metricbeat]] -. Start {metricbeat}: +. [[gs-start-metricbeat]]Start {metricbeat}: + *deb and rpm:* + @@ -531,13 +618,20 @@ standard error instead of syslog. sudo service metricbeat start ---- + -*mac:* +*mac and linux:* + [source,yaml] ---- ./metricbeat -e ---- + +*brew:* ++ +[source,yaml] +---- +metricbeat -e +---- ++ *win:* + [source,yaml] @@ -553,7 +647,7 @@ PS C:\Program Files\Metricbeat> Start-Service metricbeat ==== Visualize system metrics in {kib} To visualize system metrics, open your browser and navigate to the {metricbeat} -system overview dashboard: http://localhost:5601/app/kibana#/dashboard/Metricbeat-system-overview +system overview dashboard: http://localhost:5601/app/kibana#/dashboard/Metricbeat-system-overview-ecs TIP: If you don’t see data in {kib}, try changing the date range to a larger range. By default, {kib} shows the last 15 minutes. If you see errors, make @@ -574,12 +668,13 @@ same server--let's add {ls}. [float] [[install-logstash]] -=== Install {ls} +=== Install {ls} (optional) https://www.elastic.co/products/logstash[{ls}] is a powerful tool that integrates with a wide variety of deployments. It offers a large selection of plugins to help you parse, enrich, transform, and buffer data from a variety of -sources. +sources. If your data requires additional processing that is not available in +{beats}, then you need to add {ls} to your deployment. To download and install {ls}, open a terminal window and use the commands that work with your system: @@ -620,7 +715,7 @@ sudo rpm -i logstash-{logstash_version}.rpm endif::[] -*mac:* +*mac and linux:* ifeval::["{release-state}"=="unreleased"] @@ -638,6 +733,24 @@ tar -xzvf logstash-{logstash_version}.tar.gz endif::[] +*brew:* + +ifeval::["{release-state}"=="unreleased"] + +Version {version} of {ls} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes"] +---------------------------------------------------------------------- +brew tap elastic/tap +brew install elastic/tap/logstash-full +---------------------------------------------------------------------- + +endif::[] + *win:* ifeval::["{release-state}"=="unreleased"] @@ -668,7 +781,7 @@ To learn more about installing, configuring, and running {ls}, read the ==== Configure {ls} to listen for {beats} input {ls} provides {logstash-ref}/input-plugins.html[input plugins] for reading from -a variety of inputs. In this tutorial, you create a {ls} pipeline configuration +a variety of inputs. In this guide, you create a {ls} pipeline configuration that listens for {beats} input and sends the received events to the {es} output. To configure {ls}: @@ -719,6 +832,8 @@ enriching, and transforming data. Use the command that works with your system. If you installed {ls} as a deb or rpm package, make sure the config file is in the `config` directory. +For other platforms, a `config` directory isn't required, but it's a best +practice to be consistent. *deb:* @@ -734,18 +849,26 @@ sudo /etc/init.d/logstash start sudo service logstash start ---------------------------------------------------------------------- -*mac:* +*mac and linux:* ["source","sh",subs="attributes,callouts"] ---------------------------------------------------------------------- -./bin/logstash -f demo-metrics-pipeline.conf +cd logstash-{logstash_version} +./bin/logstash -f path/to/config/demo-metrics-pipeline.conf +---------------------------------------------------------------------- + +*brew:* + +["source","sh",subs="attributes,callouts"] +---------------------------------------------------------------------- +logstash -f path/to/config/demo-metrics-pipeline.conf ---------------------------------------------------------------------- *win:* ["source","sh",subs="attributes,callouts"] ---------------------------------------------------------------------- -bin\logstash.bat -f demo-metrics-pipeline.conf +bin\logstash.bat -f path\to\config\demo-metrics-pipeline.conf ---------------------------------------------------------------------- TIP: If you receive JVM error messages, check your Java version as shown in @@ -759,7 +882,8 @@ configure {metricbeat} to send events to {ls}. {metricbeat} sends events to {es} by default. To send events to {ls}, modify the {metricbeat} configuration file, `metricbeat.yml`. You'll find this file under -the {metricbeat} install directory, or `/etc/metricbeat` for rpm and deb. +the {metricbeat} install directory, `/etc/metricbeat` for rpm and deb, or +`/usr/local/etc/metricbeat` for brew. Disable the `output.elasticsearch` section by commenting it out, then enable the `output.logstash` section by uncommenting it: @@ -808,9 +932,9 @@ For example: Rather than sending the whole command-line argument to {es}, you might want to send just the command's path. One way to do that is by using a Grok filter. -Learning Grok is beyond the scope of this tutorial, but if you want to learn -more, see the {logstash-ref}/plugins-filters-grok.html[Grok filter plugin] -documentation. +Learning Grok is beyond the scope of this getting started guide, but if you want +to learn more, see the +{logstash-ref}/plugins-filters-grok.html[Grok filter plugin] documentation. To extract the path, add the following Grok filter between the input and output sections in the {ls} config file that you created earlier: @@ -878,6 +1002,7 @@ Restart {ls} to pick up the changes. The event now includes a field called [float] +[[whats_next]] ==== What's next? Congratulations! You've successfully set up the {stack}. You learned how to @@ -888,9 +1013,16 @@ Next, you'll want to set up the {stack} {security-features} and activate your trial license so you can unlock the full capabilities of the {stack}. To learn how, read: -* {stack-ov}/elasticsearch-security.html[Securing the {stack}] +* {ref}/elasticsearch-security.html[Securing the {stack}] * {stack-ov}/license-management.html[License Management] +Want to get up and running quickly with metrics monitoring and +centralized log analytics? Try out the Metrics app and the Logs app in {kib}. +For more details, see the {metrics-guide}[Metrics Monitoring Guide] +and the {logs-guide}[Logs Monitoring Guide]. + + + Later, when you're ready to set up a production environment, also see the {stack-ref}/installing-elastic-stack.html[{stack} Installation and Upgrade Guide]. diff --git a/docs/en/getting-started/index.asciidoc b/docs/en/getting-started/index.asciidoc index 8739ce63a..441f26b63 100644 --- a/docs/en/getting-started/index.asciidoc +++ b/docs/en/getting-started/index.asciidoc @@ -14,7 +14,7 @@ :kib-repo-dir: {docdir}/../../../../kibana/docs :xes-repo-dir: {docdir}/../../../../elasticsearch/x-pack/docs/en -include::{asciidoc-dir}/../../shared/versions.asciidoc[] +include::{asciidoc-dir}/../../shared/versions/stack/{source_branch}.asciidoc[] include::{asciidoc-dir}/../../shared/attributes.asciidoc[] include::get-started-stack.asciidoc[] diff --git a/docs/en/glossary/glossary.asciidoc b/docs/en/glossary/glossary.asciidoc deleted file mode 100644 index a02be4b99..000000000 --- a/docs/en/glossary/glossary.asciidoc +++ /dev/null @@ -1,723 +0,0 @@ -[[terms]] -== Terminology - -ifdef::logstash-terms[] - -[[glossary-metadata]] @metadata :: - -A special field for storing content that you don't want to include in output -<>. For example, the `@metadata` field is useful for -creating transient fields for use in <> -statements. -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::cloud-terms[] - -[[glossary-admin-console]] administration console :: - -A component of {ece} that provides the API server for the -<>. Also syncs cluster and allocator data from -ZooKeeper to {es}. -+ -//Source: Cloud -endif::cloud-terms[] -ifdef::cloud-terms[] - -[[glossary-allocator]] allocator :: - -Manages hosts that contain {es} and {kib} nodes. Controls the lifecycle of these -nodes by creating new <> and managing the nodes -within these containers when requested. Used to scale the capacity of your {ece} -installation. -+ -//Source: Cloud -endif::cloud-terms[] -ifdef::elasticsearch-terms[] - -[[glossary-analysis]] analysis :: - -Analysis is the process of converting <> to -<>. Depending on which analyzer is used, these phrases: -`FOO BAR`, `Foo-Bar`, `foo,bar` will probably all result in the -terms `foo` and `bar`. These terms are what is actually stored in -the index. -+ -A full text query (not a <> query) for `FoO:bAR` will -also be analyzed to the terms `foo`,`bar` and will thus match the -terms stored in the index. -+ -It is this process of analysis (both at index time and at search time) -that allows {es} to perform full text queries. -+ -Also see <> and <>. -+ -//Source: Elasticsearch -endif::elasticsearch-terms[] -ifdef::cloud-terms[] - -[[glossary-zone]] availability zone :: - -Contains resources available to a {ece} installation that are isolated from -other availability zones to safeguard against failure. Could be a rack, a server -zone or some other logical constraint that creates a failure boundary. In a -highly available cluster, the nodes of a cluster are spread across two or three -availability zones to ensure that the cluster can survive the failure of an -entire availability zone. Also see -{ece-ref}/ece-ha.html[Fault Tolerance (High Availability)]. -+ -//Source: Cloud -endif::cloud-terms[] -ifdef::cloud-terms[] - -[[glossary-beats-runner]] beats runner :: - -Used to send Filebeat and Metricbeat information to the logging cluster. -+ -//Source: Cloud -endif::cloud-terms[] -ifdef::xpack-terms[] - -[[glossary-ml-bucket]] bucket :: - -The {ml-features} use the concept of a bucket to divide the time -series into batches for processing. The _bucket span_ is part of the -configuration information for a job. It defines the time interval that is used -to summarize and model the data. This is typically between 5 minutes to 1 hour -and it depends on your data characteristics. When you set the bucket span, -take into account the granularity at which you want to analyze, the frequency -of the input data, the typical duration of the anomalies, and the frequency at -which alerting is required. -+ -//Source: X-Pack -endif::xpack-terms[] -ifdef::cloud-terms[] - -[[glossary-client-forwarder]] client forwarder :: - -Used for secure internal communications between various components of {ece} and -ZooKeeper. -+ -//Source: Cloud -endif::cloud-terms[] -ifdef::cloud-terms[] - -[[glossary-cloud-ui]] Cloud UI :: - -Provides web-based access to manage your {ece} installation, supported by the -<>. -+ -//Source: Cloud -endif::cloud-terms[] -ifdef::elasticsearch-terms,cloud-terms[] - -[[glossary-cluster]] cluster :: - -A cluster consists of one or more <> which share the -same cluster name. Each cluster has a single master node which is -chosen automatically by the cluster and which can be replaced if the -current master node fails. -+ -//Source: Elasticsearch -endif::elasticsearch-terms,cloud-terms[] -ifdef::logstash-terms[] - -[[glossary-codec-plugin]] codec plugin :: - -A Logstash <> that changes the data representation -of an <>. Codecs are essentially stream filters that -can operate as part of an input or output. Codecs enable you to separate the -transport of messages from the serialization process. Popular codecs include -json, msgpack, and plain (text). -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::logstash-terms[] - -[[glossary-conditional]] conditional :: - -A control flow that executes certain actions based on whether a statement -(also called a condition) is true or false. Logstash supports `if`, -`else if`, and `else` statements. You can use conditional statements to -apply filters and send events to a specific output based on conditions that -you specify. -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::cloud-terms[] - -[[glossary-constructor]] constructor :: - -Directs <> to manage containers of {es} and {kib} -nodes and maximizes the utilization of allocators. Monitors plan change requests -from the Cloud UI and determines how to transform the existing cluster. In a -highly available installation, places cluster nodes within different -availability zones to ensure that the cluster can survive the failure of an -entire availability zone. -+ -//Source: Cloud -endif::cloud-terms[] -ifdef::cloud-terms[] - -[[glossary-container]] container :: - -Includes an instance of {ece} software and its dependencies. Used to provision -similar environments, to assign a guaranteed share of host resources to nodes, -and to simplify operational effort in {ece}. -+ -//Source: Cloud -endif::cloud-terms[] -ifdef::cloud-terms[] - -[[glossary-coordinator]] coordinator :: - -Consists of a logical grouping of some {ece} services and acts as a distributed -coordination system and resource scheduler. -+ -//Source: Cloud -endif::cloud-terms[] -ifdef::xpack-terms[] - -[[glossary-ml-datafeed]] datafeed :: - -Machine learning jobs can analyze either a one-off batch of data or -continuously in real time. {dfeeds-cap} retrieve data from {es} for analysis. -Alternatively you can post data from any source directly to a {ml} API. -+ -//Source: X-Pack -endif::xpack-terms[] - -ifdef::xpack-terms[] - -[[glossary-ml-detector]] detector :: - -As part of the configuration information that is associated with a -{ml} job, detectors define the type of analysis that needs to be done. They -also specify which fields to analyze. You can have more than one detector in a -job, which is more efficient than running multiple jobs against the same data. -+ -//Source: X-Pack -endif::xpack-terms[] - -ifdef::cloud-terms[] - -[[glossary-director]] director :: - -Manages the <> datastore. This role is often -shared with the <>, though in production -deployments it can be separated. -+ -//Source: Cloud -endif::cloud-terms[] -ifdef::elasticsearch-terms[] - -[[glossary-document]] document :: - -A document is a JSON document which is stored in {es}. It is -like a row in a table in a relational database. Each document is -stored in an <> and has a <> and an -<>. -+ -A document is a JSON object (also known in other languages as a hash / -hashmap / associative array) which contains zero or more -<>, or key-value pairs. -+ -The original JSON document that is indexed will be stored in the -<>, which is returned by default when -getting or searching for a document. -+ -//Source: Elasticsearch -endif::elasticsearch-terms[] -ifdef::logstash-terms[] - -[[glossary-event]] event :: - -A single unit of information, containing a timestamp plus additional data. An -event arrives via an input, and is subsequently parsed, timestamped, and -passed through the Logstash <>. -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::elasticsearch-terms,logstash-terms[] - -[[glossary-field]] field :: -endif::elasticsearch-terms,logstash-terms[] -ifdef::elasticsearch-terms[] -A <> contains a list of fields, or key-value -pairs. The value can be a simple (scalar) value (for example, a string, -integer, date), or a nested structure like an array or an object. A field is -similar to a column in a table in a relational database. -+ -The <> for each field has a field _type_ (not to -be confused with document <>) which indicates the type -of data that can be stored in that field, eg `integer`, `string`, -`object`. The mapping also allows you to define (amongst other things) -how the value for a field should be analyzed. -+ -//Source: Elasticsearch -+ -endif::elasticsearch-terms[] -ifdef::logstash-terms[] -In Logstash, this term refers to an <> property. For -example, each event in an apache access log has properties, such as a status -code (200, 404), request path ("/", "index.html"), HTTP verb (GET, POST), client -IP address, and so on. Logstash uses the term "fields" to refer to these -properties. -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::logstash-terms[] - -[[glossary-field-reference]] field reference :: - -A reference to an event <>. This reference may appear in -an output block or filter block in the Logstash config file. Field references -are typically wrapped in square (`[]`) brackets, for example `[fieldname]`. If -you are referring to a top-level field, you can omit the `[]` and simply use -the field name. To refer to a nested field, you specify the full path to that -field: `[top-level field][nested field]`. -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::logstash-terms[] - -[[glossary-filter-plugin]] filter plugin :: - -A Logstash <> that performs intermediary processing on -an <>. Typically, filters act upon event data after it -has been ingested via inputs, by mutating, enriching, and/or modifying the -data according to configuration rules. Filters are often applied conditionally -depending on the characteristics of the event. Popular filter plugins include -grok, mutate, drop, clone, and geoip. Filter stages are optional. -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::logstash-terms[] - -[[glossary-gem]] gem :: - -A self-contained package of code that's hosted on -https://rubygems.org[RubyGems.org]. Logstash <> are -packaged as Ruby Gems. You can use the Logstash -<> to manage Logstash gems. -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::logstash-terms[] - -[[glossary-hot-thread]] hot thread :: - -A Java thread that has high CPU usage and executes for a longer than normal -period of time. -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::elasticsearch-terms[] - -[[glossary-id]] id :: - -The ID of a <> identifies a document. The -`index/id` of a document must be unique. If no ID is provided, -then it will be auto-generated. (Also see <>). -+ -//Source: Elasticsearch -endif::elasticsearch-terms[] -ifdef::elasticsearch-terms[] - -[[glossary-index]] index :: - -An index is like a _table_ in a relational database. It has a -<> which contains a <>, -which contains the <> in the index. -+ -An index is a logical namespace which maps to one or more -<> and can have zero or more -<>. -+ -//Source: Elasticsearch -endif::elasticsearch-terms[] -ifdef::logstash-terms[] - -[[glossary-indexer]] indexer :: - -A Logstash instance that is tasked with interfacing with an {es} cluster in -order to index <> data. -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::logstash-terms[] - -[[glossary-input-plugin]] input plugin :: - -A Logstash <> that reads <> data -from a specific source. Input plugins are the first stage in the Logstash -event processing <>. Popular input plugins include -file, syslog, redis, and beats. -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::xpack-terms[] - -[[glossary-ml-job]] job :: - -Machine learning jobs contain the configuration information and metadata -necessary to perform an analytics task. -+ -//Source: X-Pack -endif::xpack-terms[] -ifdef::xpack-terms[] - -[[glossary-ml-nodes]] -machine learning node :: - -A {ml} node is a node that has `xpack.ml.enabled` and `node.ml` set to `true`, -which is the default behavior. If you set `node.ml` to `false`, the node can -service API requests but it cannot run jobs. If you want to use {ml-features}, -there must be at least one {ml} node in your cluster. -+ -//Source: X-Pack -endif::xpack-terms[] -ifdef::elasticsearch-terms[] - -[[glossary-mapping]] mapping :: - -A mapping is like a _schema definition_ in a relational database. Each -<> has a mapping, which defines a <>, -plus a number of index-wide settings. -+ -A mapping can either be defined explicitly, or it will be generated -automatically when a document is indexed. -+ -//Source: Elasticsearch -endif::elasticsearch-terms[] -ifdef::cloud-terms[] - -[[glossary-master-node]] master node :: - -Handles write requests for the cluster and publishes changes to other nodes in -an ordered fashion. Each cluster has a single master node which is chosen -automatically by the cluster and is replaced if the current master node fails. -Also see <>. -+ -//Source: Cloud -endif::cloud-terms[] -ifdef::elasticsearch-terms[] - -[[glossary-merge]] merge :: - -The combining of Lucene segments, either automatically in the background or initiated using force merge. -+ -//Source: Elasticsearch -endif::elasticsearch-terms[] -ifdef::logstash-terms[] - -[[glossary-message-broker]] message broker :: - -Also referred to as a _message buffer_ or _message queue_, a message broker is -external software (such as Redis, Kafka, or RabbitMQ) that stores messages -from the Logstash shipper instance as an intermediate store, waiting to be -processed by the Logstash indexer instance. -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::elasticsearch-terms,cloud-terms[] - -[[glossary-node]] node :: - -A node is a running instance of {es} or {kib} which belongs to a -<>. Multiple nodes can be started on a single server -for testing purposes, but usually you should have one node per server. -+ -At startup, a node will use unicast to discover an existing cluster with -the same cluster name and will try to join that cluster. -+ -//Source: Elasticsearch -endif::elasticsearch-terms,cloud-terms[] -ifdef::logstash-terms[] - -[[glossary-output-plugin]] output plugin :: - -A Logstash <> that writes <> data -to a specific destination. Outputs are the final stage in the event -<>. Popular output plugins include elasticsearch, -file, graphite, and statsd. -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::logstash-terms[] - -[[glossary-pipeline]] pipeline :: - -A term used to describe the flow of <> through the -Logstash workflow. A pipeline typically consists of a series of input, filter, -and output stages. <> stages get data from a source -and generate events, <> stages, which are -optional, modify the event data, and <> stages -write the data to a destination. Inputs and outputs support -<> that enable you to encode or decode the data as -it enters or exits the pipeline without having to use a separate filter. -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::cloud-terms[] - -[[glossary-plan]] plan :: - -Specifies the configuration and topology of an {es} or {kib} cluster, such as -capacity, availability, and {es} version, for example. When changing a plan, the -<> determines how to transform the existing -cluster into the pending plan. -+ -//Source: Cloud -endif::cloud-terms[] -ifdef::logstash-terms[] - -[[glossary-plugin]] plugin :: - -A self-contained software package that implements one of the stages in the -Logstash event processing <>. The list of available -plugins includes <>, -<>, -<>, and -<>. The plugins are implemented as Ruby -<> and hosted on https://rubygems.org[RubyGems.org]. You -define the stages of an event processing <> -by configuring plugins. -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::logstash-terms[] - -[[glossary-plugin-manager]] plugin manager :: - -Accessed via the `bin/logstash-plugin` script, the plugin manager enables -you to manage the lifecycle of <> in your Logstash -deployment. You can install, remove, and upgrade plugins by using the -plugin manager Command Line Interface (CLI). -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::elasticsearch-terms[] - -[[glossary-primary-shard]] primary shard :: - -Each document is stored in a single primary <>. When -you index a document, it is indexed first on the primary shard, then -on all <> of the primary shard. -+ -By default, an <> has 5 primary shards. You can -specify fewer or more primary shards to scale the number of -<> that your index can handle. -+ -You cannot change the number of primary shards in an index, once the -index is created. -+ -See also <>. -+ -//Source: Elasticsearch -endif::elasticsearch-terms[] -ifdef::cloud-terms[] - -[[glossary-proxy]] proxy :: - -A highly available, TLS-enabled proxy layer that routes user requests, mapping -cluster IDs that are passed in request URLs for the container to the cluster -nodes handling the user requests. -+ -//Source: Cloud -endif::cloud-terms[] -ifdef::elasticsearch-terms[] - -[[glossary-reindex]] reindex :: - -To cycle through some or all documents in one or more indices, re-writing them into the same or new index in a local or remote cluster. This is most commonly done to update mappings, or to upgrade Elasticsearch between two incompatible index versions. -//Source: Elasticsearch -endif::elasticsearch-terms[] -ifdef::elasticsearch-terms[] - -[[glossary-replica-shard]] replica shard :: - -Each <> can have zero or more -replicas. A replica is a copy of the primary shard, and has two -purposes: -+ -1. increase failover: a replica shard can be promoted to a primary -shard if the primary fails -2. increase performance: get and search requests can be handled by -primary or replica shards. -+ -By default, each primary shard has one replica, but the number of -replicas can be changed dynamically on an existing index. A replica -shard will never be started on the same node as its primary shard. -+ -//Source: Elasticsearch -endif::elasticsearch-terms[] -ifdef::cloud-terms[] - -[[glossary-roles-token]] roles token :: - -Enables a host to join an existing {ece} installation and grants permission to -hosts to hold certain roles, such as the <> role. -Used when installing {ece} on additional hosts, a roles token helps secure {ece} -by making sure that only authorized hosts become part of the installation. -+ -//Source: Cloud -endif::cloud-terms[] -ifdef::elasticsearch-terms[] - -[[glossary-routing]] routing :: - -When you index a document, it is stored on a single -<>. That shard is chosen by hashing -the `routing` value. By default, the `routing` value is derived from -the ID of the document or, if the document has a specified parent -document, from the ID of the parent document (to ensure that child and -parent documents are stored on the same shard). -+ -This value can be overridden by specifying a `routing` value at index -time, or a {ref}/mapping-routing-field.html[routing field] in the -<>. -+ -//Source: Elasticsearch -endif::elasticsearch-terms[] -ifdef::cloud-terms[] - -[[glossary-runner]] runner :: - -A local control agent that runs on all hosts, used to deploy local containers -based on role definitions. Ensures that containers assigned to it exist and are -able to run, and creates or recreates the containers if necessary. -+ -//Source: Cloud -endif::cloud-terms[] -ifdef::cloud-terms[] - -[[glossary-services-forwarder]] services forwarder :: - -Routes data internally in an {ece} installation. -+ -//Source: Cloud -endif::cloud-terms[] -ifdef::elasticsearch-terms[] - -[[glossary-shard]] shard :: - -A shard is a single Lucene instance. It is a low-level “worker” unit -which is managed automatically by {es}. An index is a logical -namespace which points to <> and -<> shards. -+ -Other than defining the number of primary and replica shards that an -index should have, you never need to refer to shards directly. -Instead, your code should deal only with an index. -+ -{es} distributes shards amongst all <> in the -<>, and can move shards automatically from one -node to another in the case of node failure, or the addition of new -nodes. -+ -//Source: Elasticsearch -endif::elasticsearch-terms[] -ifdef::logstash-terms[] - -[[glossary-shipper]] shipper :: - -An instance of Logstash that send events to another instance of Logstash, or -some other application. -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::elasticsearch-terms[] - -[[glossary-shrink]] shrink :: - -To reduce the amount of shards in an index. See the {ref}/indices-shrink-index.html[shrink index API]. -+ -//Source: Elasticsearch -endif::elasticsearch-terms[] -ifdef::elasticsearch-terms[] - -[[glossary-source_field]] source field :: - -By default, the JSON document that you index will be stored in the -`_source` field and will be returned by all get and search requests. -This allows you access to the original object directly from search -results, rather than requiring a second step to retrieve the object -from an ID. -+ -//Source: Elasticsearch -endif::elasticsearch-terms[] -ifdef::elasticsearch-terms[] - -[[glossary-split]] split :: - -To grow the amount of shards in an index. See the {ref}/indices-split-index.html[split index API]. -//Source: Elasticsearch -endif::elasticsearch-terms[] -ifdef::cloud-terms[] - -[[glossary-stunnel]] stunnel :: - -Securely tunnels all traffic in an {ece} installation. -+ -//Source: Cloud -endif::cloud-terms[] -ifdef::elasticsearch-terms[] - -[[glossary-term]] term :: - -A term is an exact value that is indexed in {es}. The terms -`foo`, `Foo`, `FOO` are NOT equivalent. Terms (i.e. exact values) can -be searched for using _term_ queries. + -See also <> and <>. -+ -//Source: Elasticsearch -endif::elasticsearch-terms[] -ifdef::elasticsearch-terms[] - -[[glossary-text]] text :: - -Text (or full text) is ordinary unstructured text, such as this -paragraph. By default, text will be <> into -<>, which is what is actually stored in the index. -+ -Text <> need to be analyzed at index time in order to -be searchable as full text, and keywords in full text queries must be -analyzed at search time to produce (and search for) the same terms -that were generated at index time. -+ -See also <> and <>. -+ -//Source: Elasticsearch -endif::elasticsearch-terms[] -ifdef::elasticsearch-terms[] - -[[glossary-type]] type :: - -A type used to represent the _type_ of document, e.g. an `email`, a `user`, or a `tweet`. -Types are deprecated and are in the process of being removed. See -{ref}/removal-of-types.html[Removal of mapping types]. -+ -//Source: Elasticsearch -endif::elasticsearch-terms[] -ifdef::logstash-terms[] - -[[glossary-worker]] worker :: - -The filter thread model used by Logstash, where each worker receives an -<> and applies all filters, in order, before emitting -the event to the output queue. This allows scalability across CPUs because -many filters are CPU intensive. -+ -//Source: Logstash -endif::logstash-terms[] -ifdef::cloud-terms[] - -[[glossary-zookeeper]] ZooKeeper :: - -A coordination service for distributed systems used by {ece} to store the state -of the installation. Responsible for discovery of hosts, resource allocation, -leader election after failure and high priority notifications. -+ -//Source: Cloud -endif::cloud-terms[] diff --git a/docs/en/glossary/index.asciidoc b/docs/en/glossary/index.asciidoc deleted file mode 100644 index 179b7027a..000000000 --- a/docs/en/glossary/index.asciidoc +++ /dev/null @@ -1,13 +0,0 @@ -[[elastic-glossary]] -= Elastic Glossary - -:cloud-terms: true -:elasticsearch-terms: true -:logstash-terms: true -:xpack-terms: true - -include::{asciidoc-dir}/../../shared/versions.asciidoc[] -include::{asciidoc-dir}/../../shared/attributes.asciidoc[] - -include::intro.asciidoc[] -include::glossary.asciidoc[] diff --git a/docs/en/glossary/intro.asciidoc b/docs/en/glossary/intro.asciidoc deleted file mode 100644 index 61f64359c..000000000 --- a/docs/en/glossary/intro.asciidoc +++ /dev/null @@ -1,7 +0,0 @@ -[[intro]] -== Introduction - -The products in the https://www.elastic.co/products[Elastic Stack] -are designed to be used together and thus there is common terminology across -the stack. In the exceptional cases where a term has different connotations in -specific products or domains, those differences are noted in the definition. diff --git a/docs/en/infraops/index.asciidoc b/docs/en/infraops/index.asciidoc deleted file mode 100644 index 44c051d1c..000000000 --- a/docs/en/infraops/index.asciidoc +++ /dev/null @@ -1,18 +0,0 @@ -:doctype: book -:infra-ui: Infrastructure -:infra-soln: infrastructure monitoring -:logs-ui: Logs - -= Infrastructure Monitoring Guide - -include::{asciidoc-dir}/../../shared/versions.asciidoc[] - -include::{asciidoc-dir}/../../shared/attributes.asciidoc[] - -include::overview.asciidoc[] - -include::installation.asciidoc[] - -include::infra-ui-intro.asciidoc[] - -include::logs-ui-intro.asciidoc[] diff --git a/docs/en/infraops/infra-ui-intro.asciidoc b/docs/en/infraops/infra-ui-intro.asciidoc deleted file mode 100644 index c58284c7d..000000000 --- a/docs/en/infraops/infra-ui-intro.asciidoc +++ /dev/null @@ -1,49 +0,0 @@ -[[infrastructure-ui-overview]] -[role="xpack"] -== {infra-ui} UI - -beta[] - -After you have infrastructure monitoring <> -and data is streaming to {es}, use the {infra-ui} UI in {kib} to monitor your -infrastructure and identify problems in real time. - -For more information about working with the {infra-ui} UI, see the -{kibana-ref}/xpack-infra.html[{kib} documentation]. - -[float] -=== Monitor your hosts and containers - -You start with an overview of the hosts and containers running in your -infrastructure. The overview provides a summary of high-level metrics, like CPU -usage, system load, memory usage, and network traffic, to help you assess the -overall health of your systems and services. - -You can search for specific hosts to filter the overview, or enter -{kibana-ref}/kuery-query.html[Kibana Query Language] for more sophisticated -searches. To see data about related hosts or containers, you can group by important -characteristics, such as availability zones for cloud infrastructure, or -namespaces for containers. - -When you see a potential problem, you can drill down into individual nodes to -view related metrics and logs. - -[role="screenshot"] -image::screenshot-systems-monitoring.jpg[] - -[float] -=== View detailed metrics - -After drilling down into the metrics for a specific node, you see details like -CPU usage, system load, memory usage, and network traffic over time. You can -place your cursor over a point in the timeline to see detailed metrics captured -at that moment in the timeline. - -[role="screenshot"] -image::screenshot-metrics-infrastructure-monitoring.jpg[] - -[float] -=== View related logs - -You can drill down into the logs for a specific node and explore the log data -in the <>. diff --git a/docs/en/infraops/installation.asciidoc b/docs/en/infraops/installation.asciidoc deleted file mode 100644 index 0a5300140..000000000 --- a/docs/en/infraops/installation.asciidoc +++ /dev/null @@ -1,105 +0,0 @@ -[[install-infrastructure-monitoring]] -[role="xpack"] -== Get up and running - -beta[] - -To get up and running with infrastructure monitoring, you need: - -* An Elasticsearch cluster and Kibana (version 6.5 or later) with a basic -license. To learn how to get started quickly, see -{stack-gs}/get-started-elastic-stack.html[Getting started with the {stack}]. -+ -[TIP] -============== -You can skip having to install {es} and {kib} by using our -https://www.elastic.co/cloud/elasticsearch-service[hosted {es} Service] on -Elastic Cloud. The {es} Service is available on both AWS and GCP. -https://www.elastic.co/cloud/elasticsearch-service/signup[Try out the {es} -Service for free]. -============== - -* {beats} shippers (version 6.5 or later) installed on each system you want to -monitor - -You might also need to modify UI settings in {kib} to change default behaviors, -such as the index pattern used to query the data, and the timestamp field used -for sorting. For more information, see -{kibana-ref}/infrastructure-ui-settings-kb.html[{infra-ui} UI Settings] and -{kibana-ref}/logs-ui-settings-kb.html[{logs-ui} UI Settings]. - -[float] -[[install-beats-for-infra-UI]] -=== Install {beats} shippers - -To populate the <> with metrics and -log data, you need to install and configure the following shippers: - -* https://www.elastic.co/products/beats/metricbeat[{metricbeat}] for metrics -* https://www.elastic.co/products/beats/filebeat[{filebeat}] for log data - -To learn how to install and configure the shippers, do one of the following: - -* Follow the instructions in the Add Data section of the {kib} home page. Click -*Add log data* or *Add metrics*, and follow the links for the types of data you -want to collect. -+ -[role="screenshot"] -image::add-data.png[] - -* If your data source isn't in the list, or you want to install {beats} the old -fashioned way: - -** Follow the -{metricbeat-ref}/metricbeat-getting-started.html[{metricbeat} getting started] -and enable modules for the metrics you want to collect. - -** Follow the -{filebeat-ref}/filebeat-modules-quickstart.html[{filebeat} modules quick start] -and enable modules for the logs you want to collect. If there is no module -for the logs you want to collect, see the -{filebeat-ref}/filebeat-getting-started.html[{filebeat} getting started] to -learn how to configure inputs. - -For either approach, you need to enable modules in {filebeat} and {metricbeat} -to populate the {infra-ui} UI with data. - -[float] -==== Which modules and configuration options do I enable? - -To populate the *Hosts* view and add logs, enable: - -* {metricbeat-ref}/metricbeat-module-system.html[{metricbeat} `system` module] (enabled by default) -* {filebeat-ref}/filebeat-module-system.html[{filebeat} `system` module] -* {filebeat-ref}/filebeat-modules.html[Other {filebeat} modules] needed for -your environment, such as `apache2`, `redis`, and so on -* {metricbeat-ref}/add-host-metadata.html[{metricbeat} `add_host_metadata` processor] (enabled by default) -* {metricbeat-ref}/add-cloud-metadata.html[{metricbeat} `add_cloud_metadata` processor] (enabled by default) - -To populate the *Docker* view and add logs, enable: - -* {metricbeat-ref}/metricbeat-module-docker.html[{metricbeat} `docker` module] -* {metricbeat-ref}/add-docker-metadata.html[{metricbeat} `add_docker_metadata` processor] -* {filebeat-ref}/filebeat-input-docker.html[{filebeat} `docker` input] -* {filebeat-ref}/add-docker-metadata.html[{filebeat} `add_docker_metadata` processor] - -To populate the *Kubernetes* view and add logs, enable: - -* {metricbeat-ref}/metricbeat-module-kubernetes.html[{metricbeat} `kubernetes` -module] -* {metricbeat-ref}/add-kubernetes-metadata.html[{metricbeat} `add_kubernetes_metadata` processor] -* {filebeat-ref}/filebeat-input-docker.html[{filebeat} `docker` input] -* {filebeat-ref}/add-kubernetes-metadata.html[{filebeat} `add_kubernetes_metadata` processor] - -[float] -==== More about container monitoring - -If you're monitoring containers, you can use autodiscover to automatically apply -configuration changes in response to changes in your containers. To learn how, -see: - -* {filebeat-ref}/configuration-autodiscover.html[{filebeat} autodiscover -configuration] -* {metricbeat-ref}/configuration-autodiscover.html[{metricbeat} autodiscover -configuration] - diff --git a/docs/en/infraops/logs-ui-intro.asciidoc b/docs/en/infraops/logs-ui-intro.asciidoc deleted file mode 100644 index e0032768f..000000000 --- a/docs/en/infraops/logs-ui-intro.asciidoc +++ /dev/null @@ -1,20 +0,0 @@ -[[logs-ui-overview]] -[role="xpack"] -== {logs-ui} UI - -beta[] - -The {logs-ui} UI provides real-time log tailing in a compact, customizable -display. The log data is correlated with metrics in the -<>, making it easier for you to diagnose -problems. You can stream the logs in real time, or load the new data manually. -Logs are streamed from top to bottom, making it easier for you to explore the -logs in real time and see a historical view of your log data. The search bar in -the log viewer supports {kibana-ref}/kuery-query.html[Kibana Query Language]. -You can enter ad hoc or structured queries. - -For more information about working with the {logs-ui} UI, see the -{kibana-ref}/xpack-logs.html[{kib} documentation]. - -[role="screenshot"] -image::screenshot-logs-ui.jpg[] diff --git a/docs/en/infraops/overview.asciidoc b/docs/en/infraops/overview.asciidoc deleted file mode 100644 index 830b8cb60..000000000 --- a/docs/en/infraops/overview.asciidoc +++ /dev/null @@ -1,42 +0,0 @@ -[[infrastructure-monitoring-overview]] -[role="xpack"] -== Overview - -beta[] - -You can use the infrastructure monitoring capabilities of the {stack} to monitor your -infrastructure in real time and check the health of your servers, containers, -and services. - -The <> in {kib} brings together data -from a variety of sources, including logs and metrics, making it easier for you -to identify and resolve infrastructure issues quickly. - -[float] -=== Infrastructure monitoring components - -Infrastructure monitoring requires the following {stack} components. - -image::monitoring-architecture.png[] - -*https://www.elastic.co/products/beats[{beats}]* are open source data -shippers that you install as agents on your servers to send operational data to -{es}. - -*https://www.elastic.co/products/elasticsearch[{es}]* is a real-time, -distributed storage, search, and analytics engine. {es} excels is indexing -streams of semi-structured data, such as logs or metrics. - -*https://www.elastic.co/products/kibana[{kib}]* is an open source analytics and -visualization platform designed to work with {es}. You use {kib} to search, -view, and interact with data stored in {es} indices. You can easily perform -advanced data analysis and visualize your data in a variety of charts, tables, -and maps. - -<> provides a -dedicated user interface for visualizing the health of your servers, -containers, and services. - -<> provides a compact, -customizable display for real-time log tailing. - diff --git a/docs/en/infraops/screenshot-logs-ui.jpg b/docs/en/infraops/screenshot-logs-ui.jpg deleted file mode 100644 index 567c2ee81..000000000 Binary files a/docs/en/infraops/screenshot-logs-ui.jpg and /dev/null differ diff --git a/docs/en/infraops/screenshot-metrics-infrastructure-monitoring.jpg b/docs/en/infraops/screenshot-metrics-infrastructure-monitoring.jpg deleted file mode 100755 index fc35af10e..000000000 Binary files a/docs/en/infraops/screenshot-metrics-infrastructure-monitoring.jpg and /dev/null differ diff --git a/docs/en/infraops/screenshot-systems-monitoring.jpg b/docs/en/infraops/screenshot-systems-monitoring.jpg deleted file mode 100755 index 36afcb820..000000000 Binary files a/docs/en/infraops/screenshot-systems-monitoring.jpg and /dev/null differ diff --git a/docs/en/install-upgrade/breaking.asciidoc b/docs/en/install-upgrade/breaking.asciidoc new file mode 100644 index 000000000..a01672817 --- /dev/null +++ b/docs/en/install-upgrade/breaking.asciidoc @@ -0,0 +1,109 @@ +[[elastic-stack-breaking-changes]] +== Breaking changes + +Before you upgrade, you must review the breaking changes for each product you +use and make the necessary changes so your code is compatible with {version}. + +** <> +** <> +** <> +** <> +** <> +** <> + +[IMPORTANT] +=============================== + +* If you're upgrading from 5.n, make sure you check the breaking changes from +5.n to 6.n, as well as from 6.n to 7.n! +* If you are using {ml} {dfeeds} that contain discontinued search or query +domain specific language (DSL), the upgrade will fail. In 5.6.5 and later, the +Upgrade Assistant provides information about which {dfeeds} need to be updated. + +=============================== + +[[apm-breaking-changes]] +=== APM breaking changes +++++ +APM +++++ + +coming::[7.8.0] + +This list summarizes the most important breaking changes in APM. +For the complete list, go to {apm-server-ref-70}/breaking-changes.html[APM Server 7.0 breaking changes]. + +//include::{apm-repo-dir}/apm-breaking-changes.asciidoc[tag=notable-v7-breaking-changes] + +[[beats-breaking-changes]] +=== Beats breaking changes +++++ +Beats +++++ + +coming::[7.8.0] + +This list summarizes the most important breaking changes in Beats. For the +complete list, go to {beats-ref}/breaking-changes.html[Beats breaking changes]. + +include::{beats-repo-dir}/release-notes/breaking/breaking-7.8.asciidoc[tag=notable-breaking-changes] + + +[[elasticsearch-breaking-changes]] +=== {es} breaking changes +[subs="attributes"] +++++ +{es} +++++ + +coming::[7.8.0] + +This list summarizes the most important breaking changes in {es} {version}. For +the complete list, go to {ref}/breaking-changes.html[{es} breaking changes]. + +include::{es-repo-dir}/reference/migration/migrate_7_8.asciidoc[tag=notable-breaking-changes] + +[[elasticsearch-hadoop-breaking-changes]] +=== {es} Hadoop breaking changes +[subs="attributes"] +++++ +{es} Hadoop +++++ + +coming::[7.8.0] + +This list summarizes the most important breaking changes in {es} Hadoop {version}. +For the complete list, go to +{hadoop-ref}/breaking-changes.html[Elasticsearch Hadoop breaking changes]. + +include::{hadoop-repo-dir}/appendix/breaking.adoc[tag=notable-v7-breaking-changes] + +[[kibana-breaking-changes]] +=== {kib} breaking changes +[subs="attributes"] +++++ +{kib} +++++ + +coming::[7.8.0] + +This list summarizes the most important breaking changes in {kib} {version}. For +the complete list, go to {kibana-ref}/breaking-changes.html[{kib} breaking changes]. + +//include::{kib-repo-dir}/migration/migrate_7_8.asciidoc[tag=notable-breaking-changes] + +[[logstash-breaking-changes]] +=== {ls} breaking changes +[subs="attributes"] +++++ +{ls} +++++ + +coming::[7.8.0] + +This list summarizes the most important breaking changes in {ls} {version}. For +the complete list, go to +{logstash-ref}/breaking-changes.html[Logstash breaking changes]. + +include::{ls-repo-dir}/static/breaking-changes.asciidoc[tag=notable-breaking-changes] + diff --git a/docs/en/install-upgrade/highlights.asciidoc b/docs/en/install-upgrade/highlights.asciidoc new file mode 100644 index 000000000..7185e6f68 --- /dev/null +++ b/docs/en/install-upgrade/highlights.asciidoc @@ -0,0 +1,74 @@ +[[elastic-stack-highlights]] +== Highlights + +Each release brings new features and product improvements. This section +highlights notable new features and enhancements in {version}. + +//** <> +** <> +** <> +** <> + +//// +[[apm-highlights]] +=== APM highlights +++++ +APM +++++ + +This list summarizes the most important enhancements in APM. +For the complete list, go to +{apm-overview-ref-v}/apm-release-notes.html[APM release highlights]. + +include::{apm-repo-dir}/apm-release-notes.asciidoc[tag=notable-v7-highlights] +//// + +[[beats-highlights]] +=== Beats highlights +++++ +Beats +++++ + +coming::[7.8.0] + +This list summarizes the most important enhancements in Beats. +For the complete list, go to {beats-ref}/release-highlights.html[Beats release highlights]. + +include::{beats-repo-dir}/release-notes/highlights/highlights-7.8.0.asciidoc[tag=notable-highlights] + +[[elasticsearch-highlights]] +=== {es} highlights +[subs="attributes"] +++++ +{es} +++++ + +coming::[7.8.0] + +This list summarizes the most important enhancements in {es} {version}. +For the complete list, go to {ref}/release-highlights.html[{es} release highlights]. + +:leveloffset: +1 + +include::{es-repo-dir}/reference/release-notes/highlights-7.8.0.asciidoc[tag=notable-highlights] + +:leveloffset: -1 + +[[kibana-higlights]] +=== {kib} highlights +[subs="attributes"] +++++ +{kib} +++++ + +coming::[7.8.0] + +This list summarizes the most important enhancements in {kib} {version}. + +//// +:leveloffset: +1 + +include::{kib-repo-dir}/release-notes/highlights-7.3.0.asciidoc[tag=notable-highlights] + +:leveloffset: -1 +//// \ No newline at end of file diff --git a/docs/en/install-upgrade/index.asciidoc b/docs/en/install-upgrade/index.asciidoc new file mode 100644 index 000000000..09af6b454 --- /dev/null +++ b/docs/en/install-upgrade/index.asciidoc @@ -0,0 +1,22 @@ +[[elastic-stack]] += Installation and Upgrade Guide + +:apm-repo-dir: {docdir}/../../../../apm-server/docs/guide +:beats-repo-dir: {docdir}/../../../../beats/libbeat/docs +:es-repo-dir: {docdir}/../../../../elasticsearch/docs +:hadoop-repo-dir: {docdir}/../../../../elasticsearch-hadoop/docs/src/reference/asciidoc +:kib-repo-dir: {docdir}/../../../../kibana/docs +:ls-repo-dir: {docdir}/../../../../logstash/docs + +include::{asciidoc-dir}/../../shared/versions/stack/{source_branch}.asciidoc[] +include::{asciidoc-dir}/../../shared/attributes.asciidoc[] + +include::overview.asciidoc[] + +include::installing-stack.asciidoc[] + +include::upgrading-stack.asciidoc[] + +include::highlights.asciidoc[] + +include::breaking.asciidoc[] diff --git a/docs/installing-stack.asciidoc b/docs/en/install-upgrade/installing-stack.asciidoc similarity index 93% rename from docs/installing-stack.asciidoc rename to docs/en/install-upgrade/installing-stack.asciidoc index ece8064c4..17c07c455 100644 --- a/docs/installing-stack.asciidoc +++ b/docs/en/install-upgrade/installing-stack.asciidoc @@ -8,6 +8,7 @@ Kibana {version}, and Logstash {version}. If you're upgrading an existing installation, see <> for information about how to ensure compatibility with {version}. +[discrete] [[install-order-elastic-stack]] === Installation Order @@ -17,12 +18,13 @@ Install the Elastic Stack products you want to use in the following order: . Kibana ({kibana-ref}/install.html[install]) . Logstash ({logstash-ref}/installing-logstash.html[install]) . Beats ({beats-ref}/getting-started.html[install instructions]) -. APM Server ({apm-server-ref}/installing.html[install instructions]) +. APM Server ({apm-server-ref-70}/installing.html[install instructions]) . Elasticsearch Hadoop ({hadoop-ref}/install.html[install instructions]) Installing in this order ensures that the components each product depends on are in place. +[discrete] [[install-elastic-stack-for-elastic-cloud]] === Installing on Elastic Cloud @@ -41,7 +43,7 @@ link:https://www.elastic.co/cloud/as-a-service/subscriptions[subscription level] For example, installing custom plugins, dictionaries, and scripts requires a Gold or Platinum subscription. -You can https://www.elastic.co/cloud/elasticsearch-service/signup[try out the +You can {ess-trial}[try out the Elasticsearch Service for free]. For more information, see {cloud}/ec-getting-started.html[Getting Started with Elastic Cloud]. diff --git a/docs/index.asciidoc b/docs/en/install-upgrade/overview.asciidoc similarity index 74% rename from docs/index.asciidoc rename to docs/en/install-upgrade/overview.asciidoc index ff4518100..45577664c 100644 --- a/docs/index.asciidoc +++ b/docs/en/install-upgrade/overview.asciidoc @@ -1,9 +1,4 @@ -[[elastic-stack]] -= Elastic Stack - -include::{asciidoc-dir}/../../shared/attributes.asciidoc[] -include::{docdir}/../../elasticsearch/docs/Versions.asciidoc[] - +[[overview]] == Overview The products in the https://www.elastic.co/products[{stack}] @@ -12,7 +7,7 @@ to simplify the installation and upgrade process. The full stack consists of: * {beats-ref}/index.html[Beats {branch}] -* {apm-server-ref}/index.html[APM Server {branch}] +* {apm-server-ref-70}/index.html[APM Server {branch}] * {ref}/index.html[Elasticsearch {branch}] * {hadoop-ref}/index.html[Elasticsearch Hadoop {branch}] * {kibana-ref}/index.html[Kibana {branch}] @@ -26,10 +21,3 @@ to prepare for a stack upgrade. For detailed information about breaking changes in {version} and install and upgrade instructions for specific components, see the individual product reference guides. - -include::installing-stack.asciidoc[] - -include::upgrading-stack.asciidoc[] - - - diff --git a/docs/en/install-upgrade/upgrading-stack.asciidoc b/docs/en/install-upgrade/upgrading-stack.asciidoc new file mode 100644 index 000000000..0b8f1fb76 --- /dev/null +++ b/docs/en/install-upgrade/upgrading-stack.asciidoc @@ -0,0 +1,130 @@ +[[upgrading-elastic-stack]] +== Upgrading the Elastic Stack + +When upgrading to a new version of {es}, you need to upgrade +each of the products in your Elastic Stack. Beats and Logstash 6.7 are +compatible with {es} {version} to give you flexibility in scheduling +the upgrade. + +{es} supports rolling upgrades between minor versions, from {es} 5.6 to 6.8, +and from 6.8 to {version}. + +IMPORTANT: 5.x indices are not compatible with {version}. You must +remove or reindex them to upgrade to {version}. The default Beats and +Logstash mapping templates also need to be updated to work with {version}. + +[discrete] +=== Preparing to upgrade + +Before upgrading the Elastic Stack to {version}: + +. Check the {es} {ref}/logging.html#deprecation-logging[deprecation log] +to see if you're using any deprecated features and update your code accordingly. +By default, deprecation warnings are logged when the log level is set to `WARN`. + +. Review the <> and upgrade your code to work +with {version}. + +. Upgrade to 6.8 and use the {kibana-ref}/upgrade-assistant.html[{kib} Upgrade Assistant] to {ref}/docs-reindex.html[reindex] +any indices that are not compatible with {version}. ++ +[role="xpack"] +.Upgrade Assistant +****** +The Upgrade Assistant and migration APIs are enabled with both the Basic and +Trial licenses. You can install the default distribution of 6.8 to use the +Upgrade Assistant to prepare to upgrade even if you are upgrading to the OSS +distribution of {version}. +****** + +. Use the Upgrade Assistant to identify any changes you need to make to your +cluster configuration. + +[discrete] +[[upgrade-order-elastic-stack]] +=== Upgrade process + +When you've made the necessary changes and are ready to upgrade from 6.8 to +{version}: + +. Test the upgrade in a dev environment *before* upgrading your +production cluster. + +. Back up your data. You **cannot roll back** to an earlier version unless +you have a snapshot of your data. For information about creating snapshots, see +{ref}/snapshot-restore.html[Snapshot and restore]. + +. Consider closing {ml} jobs before you start the upgrade process. While {ml} +jobs can continue to run during a rolling upgrade, it increases the overhead +on the cluster during the upgrade process. For more information, see +{ref}/rolling-upgrades.html[Rolling upgrades]. + +. Upgrade the components of your Elastic Stack in the following order: + +.. {es} Hadoop: {hadoop-ref}/install.html[install instructions] +.. {es}: {ref}/setup-upgrade.html[upgrade instructions] +.. Kibana: {kibana-ref}/upgrade.html[upgrade instructions] +.. Logstash: {logstash-ref}/upgrading-logstash.html[upgrade instructions] +.. Beats: {beats-ref}/upgrading.html[upgrade instructions] +.. APM Server: {apm-server-ref-70}/upgrading.html[upgrade instructions] + +NOTE: Logstash 6.8 and Beats 6.8 are compatible with all 7.x versions of +{es}. This provides flexibility in when you schedule the upgrades +for your Logstash instances and Beats agents, but we recommend upgrading as +soon as possible to take advantage of performance improvements +and other enhancements. + +[discrete] +[[oss-stack-upgrade]] +=== Upgrading from 6.6 or earlier + +To upgrade directly to {es} {version} from versions 6.0-6.6, you must +{ref}/reindex-upgrade.html[manually reindex] any 5.x indices you need to +carry forward, and perform a {ref}/restart-upgrade.html[full cluster restart]. +This includes any internal indices created in 5.x, such as the `.kibana` and +`.security*` indices. + +Make sure all 5.x indices have been deleted before upgrading to {version}. {es} +{version} will fail to start if any 5.x indices are present. + +If you are running a version prior to 6.0, +https://www.elastic.co/guide/en/elastic-stack/6.8/upgrading-elastic-stack.html[upgrade to 6.8] +and reindex your old indices or bring up a new {version} cluster and +{ref}/reindex-upgrade-remote.html[reindex from remote]. + +The recommended path is to upgrade to 6.8 before upgrading to {version}. This +makes it easier to identify the changes you need to make to upgrade and enables +you to perform a rolling upgrade with no downtime. + +[discrete] +[[upgrade-elastic-stack-for-elastic-cloud]] +=== Upgrading on Elastic Cloud + +A single click in the Elastic Cloud console can upgrade a cluster to a newer +version, add more processing capacity, change plugins, and enable or disable +high availability, all at the same time. During the upgrade process, +{es}, Kibana, {xpack} and the officially included plugins are +upgraded simultaneously. + +Although upgrading your Elastic Cloud clusters is easy, you still need to +address breaking changes that affect your application. Minor version upgrades, +upgrades from 6.8 to {version}, and all other cluster configuration +changes can be performed with no downtime. + +To avoid downtime when a full cluster restart is required: + +. Provision an additional cluster with the new {es} version, reindex +your data, and send index requests to both clusters temporarily. + +. Verify that the new cluster performs as expected, fix any problems, and then +permanently swap in the new cluster. + +. Delete the old cluster to stop incurring additional costs. You are billed +only for the time that the new cluster runs in parallel with your old cluster. +Usage is billed on an hourly basis. + +To learn more about the upgrade process on Elastic Cloud, see +{cloud}/ec-upgrade-deployment.html[Upgrade versions]. + +NOTE: Elastic Cloud only supports upgrades to released versions. Preview +releases and master snapshots are not supported. diff --git a/docs/en/infraops/add-data.png b/docs/en/logs/images/add-data.png similarity index 100% rename from docs/en/infraops/add-data.png rename to docs/en/logs/images/add-data.png diff --git a/docs/en/logs/images/logs-console.png b/docs/en/logs/images/logs-console.png new file mode 100644 index 000000000..5feb3d960 Binary files /dev/null and b/docs/en/logs/images/logs-console.png differ diff --git a/docs/en/infraops/monitoring-architecture.png b/docs/en/logs/images/logs-monitoring-architecture.png similarity index 100% rename from docs/en/infraops/monitoring-architecture.png rename to docs/en/logs/images/logs-monitoring-architecture.png diff --git a/docs/en/logs/index.asciidoc b/docs/en/logs/index.asciidoc new file mode 100644 index 000000000..df4337c2d --- /dev/null +++ b/docs/en/logs/index.asciidoc @@ -0,0 +1,17 @@ +:doctype: book +:metrics: metrics +:metrics-app: Metrics app +:logs: logs +:logs-app: Logs app + += Logs Monitoring Guide + +include::{asciidoc-dir}/../../shared/versions/stack/{source_branch}.asciidoc[] + +include::{asciidoc-dir}/../../shared/attributes.asciidoc[] + +include::logs-overview.asciidoc[] + +include::logs-installation.asciidoc[] + +include::logs-ui-intro.asciidoc[] diff --git a/docs/en/logs/logs-installation.asciidoc b/docs/en/logs/logs-installation.asciidoc new file mode 100644 index 000000000..3ab1f607a --- /dev/null +++ b/docs/en/logs/logs-installation.asciidoc @@ -0,0 +1,92 @@ +[[install-logs-monitoring]] +[role="xpack"] +== Setting up logs monitoring + +To set up logs monitoring, you need: + +* An Elasticsearch cluster and Kibana (version 6.5 or later) with a basic license +// Add a link to what constitutes a basic license. And is this any different for the cloud? + +* Appropriate {beats} shippers (version 6.5 or later) installed and enabled on each system you want to +monitor + +If your data uses nonstandard fields, you may also need to modify some default configuration settings. + +[float] +=== Get Elasticsearch and Kibana + +To get started, you can use our hosted {es} Service on Elastic Cloud (recommended for new users), or you can install {es} and {kib} locally. + +[float] +==== Use our hosted service + +The hosted {es} Service is available on both AWS and GCP. +{ess-trial}[Try out the {es} Service for free]. + +[float] +==== Install {es} and {kib} locally + +Alternatively, you can {stack-gs}/get-started-elastic-stack.html[install {es} and {kib} locally]. +Follow the instructions to install {es}, and to install and start {kib}. + +[float] +=== Install {beats} shippers + +To start collecting logs data, you need to install and configure the {filebeat} {beats} shipper. + +You can install and configure {beats} shippers for most kinds of data directly from {kib}, or you can install {beats} yourself. + +[float] +==== Install {beats} from {kib} + +To install {beats} from {kib}, on the machine where you want to collect the data, open a {kib} browser window. +In the *Add Data to Kibana* section, click *Add log data*. +Now follow the instructions for the type of data you want to collect. +The instructions walk you through the steps required to download, install and configure the appropriate Beats modules for your data. + +[role="screenshot"] +image::images/add-data.png[] + +[float] +==== Install {beats} yourself + +If your data source doesn't have a {beats} module, or if you want to install {beats} the old fashioned way, follow the instructions in {filebeat-ref}/filebeat-modules-quickstart.html[{filebeat} modules quick start] and enable modules for the logs you want to collect. +If there is no module for the logs you want to collect, see the {filebeat-ref}/filebeat-getting-started.html[{filebeat} getting started] to learn how to configure inputs. + +[float] +=== Enable modules + +However you install {beats}, you need to enable the appropriate modules in {filebeat} to start collecting logs data. + +// ++ I think some of this is still necessary even if you've followed the instructions to install Beats from Kibana. +// ++ The instructions there explain how to enable the module. Below, we enable more stuff. +// ++ What about if you are using Cloud? Is anything different? + +To collect logs from your host system, enable: + +* {filebeat-ref}/filebeat-module-system.html[{filebeat} `system` module] +* {filebeat-ref}/filebeat-modules.html[Other {filebeat} modules] needed for your environment, such as `apache2`, `redis`, and so on + +To collect logs from Docker containers, enable: + +* {filebeat-ref}/filebeat-input-docker.html[{filebeat} `docker` input] +* {filebeat-ref}/add-docker-metadata.html[{filebeat} `add_docker_metadata` processor] + +To collect logs from Kubernetes pods, enable: + +* {filebeat-ref}/filebeat-input-docker.html[{filebeat} `docker` input] +* {filebeat-ref}/add-kubernetes-metadata.html[{filebeat} `add_kubernetes_metadata` processor] + +[float] +=== Configure your data sources + +If your logs data has nonstandard fields, you may need to modify some configuration settings in {kib}, such as the index pattern used to query the data, and the timestamp field used for sorting. +You can use the {kibana-ref}/xpack-logs-configuring.html[*Settings* tab in the {logs-app}] to modify some common settings. +Alternatively, see {kibana-ref}/logs-ui-settings-kb.html[{logs} settings] for a complete list of logs configuration settings. + +[float] +=== More about container monitoring + +If you're monitoring Docker containers or Kubernetes pods, you can use autodiscovery to automatically change the configuration settings in response to changes in your containers. +This ensures you don't stop collecting data when your container configuration changes. +To learn how to do this, see {filebeat-ref}/configuration-autodiscover.html[{filebeat} autodiscover configuration]. \ No newline at end of file diff --git a/docs/en/logs/logs-overview.asciidoc b/docs/en/logs/logs-overview.asciidoc new file mode 100644 index 000000000..060989be3 --- /dev/null +++ b/docs/en/logs/logs-overview.asciidoc @@ -0,0 +1,34 @@ +[[logs-app-overview]] +[role="xpack"] +== Logs monitoring overview + +Logs monitoring enables you to view logs from your infrastructure and identify problems in near real time. +You can view logs from servers, containers, services, and so on. +Then you can drill down to view more detailed information about an individual log entry, or you can seamlessly switch to view corresponding metrics, uptime information or APM traces where available. You can also use machine learning to automatically detect some kinds of log anomalies. + +// Add links to metrics, uptime and APM when I have good places to link to. + +[float] +=== Logs monitoring components + +image::images/logs-monitoring-architecture.png[] + +// Redo image for metrics and logs separately. + +Logs monitoring requires the following {stack} components. + +*https://www.elastic.co/products/elasticsearch[{es}]* is a real-time, +distributed storage, search, and analytics engine. +{es} can store, search, and analyze large volumes of data in near real-time. +The {logs-app} uses {es} to store logs data in {es} documents which are queried on demand. + +*https://www.elastic.co/products/beats[{beats}]* are open source data shippers that you install as agents on your servers to send data to {es}. +The {logs-app} uses Filebeat to collect logs from the servers, containers, and other services in your infrastructure. +Filebeat modules are available for most common servers, containers and services. + +*https://www.elastic.co/products/kibana[{kib}]* is an open source analytics and visualization platform designed to work with {es}. +You use {kib} to search, view, and interact with the logs data stored in {es}. +You can easily perform advanced data analysis and visualize your data in a variety of charts, tables, +and maps. +The <> in {kib} provides a dedicated user interface to view logs from the servers, containers and services in your infrastructure. + diff --git a/docs/en/logs/logs-ui-intro.asciidoc b/docs/en/logs/logs-ui-intro.asciidoc new file mode 100644 index 000000000..133ea00a8 --- /dev/null +++ b/docs/en/logs/logs-ui-intro.asciidoc @@ -0,0 +1,17 @@ +[[logs-ui-overview]] +[role="xpack"] +== {logs-app} + +After you have set up logs ingestion and log data is streaming to {es}, you can view real-time and historic logs in a compact, customizable display. +The log data is correlated with metrics data in the {metrics-guide}/infrastructure-ui-overview.html[{metrics-app}], making it easier for you to diagnose problems. +You can also view related application traces or uptime information where available. + +You can stream the logs in real time, or view historical logs from a specified time range. + +The search bar in the log viewer supports {kibana-ref}/kuery-query.html[Kibana Query Language]. +You can enter ad hoc or structured queries to filter the log entries. + +For more information see the {kibana-ref}/xpack-logs.html[{logs-app}]. + +[role="screenshot"] +image::images/logs-console.png[] diff --git a/docs/en/metrics/aws-ec2-metricset.asciidoc b/docs/en/metrics/aws-ec2-metricset.asciidoc new file mode 100644 index 000000000..9f21c54d4 --- /dev/null +++ b/docs/en/metrics/aws-ec2-metricset.asciidoc @@ -0,0 +1,22 @@ +[[aws-ec2-metricset]] +[role="xpack"] + +=== AWS EC2 Instance Fields + +*ID*:: `cloud.instance.id` +*Name*:: `cloud.instance.name` +*IP Address*:: `aws.ec2.instance.public.ip` + +[float] +=== AWS ECS Instance Metrics + +*CPU Usage*:: Average of `aws.ec2.cpu.total.pct` + +*Inbound Traffic*:: Average of `aws.ec2.network.in.bytes_per_sec` + +*Outbound Traffic*:: Average of `aws.ec2.network.out.bytes_per_sec` + +*Disk Reads (Bytes)*:: Average of `aws.ec2.diskio.read.bytes_per_sec` + +*Disk Writes (Bytes)*:: Average of `aws.ec2.diskio.write.bytes_per_sec` + diff --git a/docs/en/metrics/aws-rds-metricset.asciidoc b/docs/en/metrics/aws-rds-metricset.asciidoc new file mode 100644 index 000000000..343232525 --- /dev/null +++ b/docs/en/metrics/aws-rds-metricset.asciidoc @@ -0,0 +1,21 @@ +[[aws-rds-metricset]] +[role="xpack"] + +=== AWS RDS Database Fields + +*ID*:: `aws.rds.db_instance.arn` +*Name*:: `aws.rds.db_instance.identifier` + +[float] +=== AWS RDS Database Metrics + +*CPU Usage*:: Average of `aws.rds.cpu.total.pct` + +*Connections*:: Average of `aws.rds.database_connections` + +*Queries Executed*:: Average of `aws.rds.queries` + +*Active Transactions*:: Average of `aws.rds.transactions.active` + +*Latency*:: Average of `aws.rds.latency.dml` + diff --git a/docs/en/metrics/aws-s3-metricset.asciidoc b/docs/en/metrics/aws-s3-metricset.asciidoc new file mode 100644 index 000000000..7ffeb9ebe --- /dev/null +++ b/docs/en/metrics/aws-s3-metricset.asciidoc @@ -0,0 +1,21 @@ +[[aws-s3-metricset]] +[role="xpack"] + +=== AWS S3 Bucket Fields + +*ID*:: `aws.s3.bucket.name` +*Name*:: `aws.s3.bucket.name` + +[float] +=== AWS S3 Bucket Metrics + +*Bucket Size*:: Average of `aws.s3_daily_storage.bucket.size.bytes` + +*Total Requests*:: Average of `aws.s3_request.requests.total` + +*Number of Objects*:: Average of `aws.s3_daily_storage.number_of_objects` + +*Downloads (Bytes)*:: Average of `aws.s3_request.downloaded.bytes` + +*Uploads (Bytes)*:: Average of `aws.s3_request.uploaded.bytes` + diff --git a/docs/en/metrics/aws-sqs-metricset.asciidoc b/docs/en/metrics/aws-sqs-metricset.asciidoc new file mode 100644 index 000000000..0478cb968 --- /dev/null +++ b/docs/en/metrics/aws-sqs-metricset.asciidoc @@ -0,0 +1,22 @@ +[[aws-sqs-metricset]] +[role="xpack"] + +=== AWS SQS Queue Fields + +*ID*:: `aws.sqs.queue.name` +*Name*:: `aws.sqs.queue.name` + +[float] +=== AWS SQS Queue Metrics + +*Messages Available*:: Max of `aws.sqs.messages.visible` + +*Messages Delayed*:: Max of `aws.sqs.messages.delayed` + +*Messages Added*:: Max of `aws.sqs.messages.sent` + +*Messages Returned Empty*:: Max of `aws.sqs.messages.not_visible` + +*Oldest Message*:: Max of `aws.sqs.oldest_message_age.sec` + + diff --git a/docs/en/metrics/docker-metricset.asciidoc b/docs/en/metrics/docker-metricset.asciidoc new file mode 100644 index 000000000..8c9330c8a --- /dev/null +++ b/docs/en/metrics/docker-metricset.asciidoc @@ -0,0 +1,19 @@ +[[docker-metricset]] +[role="xpack"] + +=== Docker Container Fields + +*ID*:: `container.id` +*Name*:: `container.name` +*IP Address*:: `container.ip_address` + +[float] +=== Docker Container Metrics + +*CPU Usage*:: Average of `docker.cpu.total.pct` + +*Memory Usage*:: Average of `docker.memory.usage.pct` + +*Inbound Traffic*:: Derivative of the maximum of `docker.network.in.bytes` scaled to a 1 second rate + +*Outbound Traffic*:: Derivative of the maximum of `docker.network.out.bytes` scaled to a 1 second rate diff --git a/docs/en/metrics/host-metricset.asciidoc b/docs/en/metrics/host-metricset.asciidoc new file mode 100644 index 000000000..1315d3afc --- /dev/null +++ b/docs/en/metrics/host-metricset.asciidoc @@ -0,0 +1,24 @@ +[[host-metricset]] +[role="xpack"] + +=== Hosts Fields + +*ID*:: `host.name` +*Name*:: `host.name` +*IP Address*:: `host.ip` + +[float] +=== Host Metrics + +*CPU Usage*:: Average of `system.cpu.user.pct` added to the average of `system.cpu.system.pct` divided by `system.cpu.cores` + +*Memory Usage*:: Average of `system.memory.actual.used.pct` + +*Load*:: Average of `system.load.5` + +*Inbound Traffic*:: Derivative of the maximum of `system.network.in.bytes` scaled to a 1 second rate + +*Outbound Traffic*:: Derivative of the maximum of `system.network.out.bytes` scaled to a 1 second rate + +*Log Rate*:: Derivative of the cumulative sum of the document count scaled to a 1 second rate. +This metric relies on the same indices as the logs. diff --git a/docs/en/metrics/images/add-data.png b/docs/en/metrics/images/add-data.png new file mode 100644 index 000000000..163b66b4d Binary files /dev/null and b/docs/en/metrics/images/add-data.png differ diff --git a/docs/en/metrics/images/infra-sysmon.png b/docs/en/metrics/images/infra-sysmon.png new file mode 100644 index 000000000..5b82d8c9b Binary files /dev/null and b/docs/en/metrics/images/infra-sysmon.png differ diff --git a/docs/en/secops/images/secops-architecture.png b/docs/en/metrics/images/metrics-monitoring-architecture.png similarity index 100% rename from docs/en/secops/images/secops-architecture.png rename to docs/en/metrics/images/metrics-monitoring-architecture.png diff --git a/docs/en/metrics/index.asciidoc b/docs/en/metrics/index.asciidoc new file mode 100644 index 000000000..24bf6225b --- /dev/null +++ b/docs/en/metrics/index.asciidoc @@ -0,0 +1,19 @@ +:doctype: book +:metrics: metrics +:metrics-app: Metrics app +:logs: logs +:logs-app: Logs app + += Metrics Monitoring Guide + +include::{asciidoc-dir}/../../shared/versions/stack/{source_branch}.asciidoc[] + +include::{asciidoc-dir}/../../shared/attributes.asciidoc[] + +include::metrics-overview.asciidoc[] + +include::installation.asciidoc[] + +include::infrastructure-metrics.asciidoc[] + +include::infra-ui-intro.asciidoc[] diff --git a/docs/en/metrics/infra-ui-intro.asciidoc b/docs/en/metrics/infra-ui-intro.asciidoc new file mode 100644 index 000000000..2d57dfda1 --- /dev/null +++ b/docs/en/metrics/infra-ui-intro.asciidoc @@ -0,0 +1,12 @@ +[[infrastructure-ui-overview]] +[role="xpack"] +== {metrics-app} + +After you have <> and data is streaming to {es}, +you can use the {metrics-app} to view metrics from your infrastructure and identify problems in real time. +You can also seamlessly switch to view the corresponding logs, application traces or uptime information for a component. + +For more information see {kibana-ref}/xpack-infra.html[{metrics-app}]. + +[role="screenshot"] +image::images/infra-sysmon.png[] \ No newline at end of file diff --git a/docs/en/metrics/infrastructure-metrics.asciidoc b/docs/en/metrics/infrastructure-metrics.asciidoc new file mode 100644 index 000000000..4f8db6374 --- /dev/null +++ b/docs/en/metrics/infrastructure-metrics.asciidoc @@ -0,0 +1,35 @@ +[[infrastructure-metrics]] +[role="xpack"] + +== Infrastructure metrics + +This section contains detailed information about each of the metricsets the {metrics-app} supports. The metrics listed below are provided by the {beats} shippers. + +* <> +* <> +* <> +* <> +* <> +* <> +* <> + +[float] +=== Additional field details + +The `event.dataset` field is required to display data properly in some views. This field is a combination of `metricset.module`, which is the Metricbeat module name, and `metricset.name`, which is the metricset name. + +All the charts use the `metricset.period` to determine the optimal time interval for each metric. If `metricset.period` is not available then it falls back to 1 minute intervals. + +include::host-metricset.asciidoc[] + +include::docker-metricset.asciidoc[] + +include::kubernetes-metricset.asciidoc[] + +include::aws-ec2-metricset.asciidoc[] + +include::aws-s3-metricset.asciidoc[] + +include::aws-sqs-metricset.asciidoc[] + +include::aws-rds-metricset.asciidoc[] diff --git a/docs/en/metrics/installation.asciidoc b/docs/en/metrics/installation.asciidoc new file mode 100644 index 000000000..7aa9455ff --- /dev/null +++ b/docs/en/metrics/installation.asciidoc @@ -0,0 +1,92 @@ +[[install-infrastructure-monitoring]] +[role="xpack"] +== Setting up {metrics} monitoring + +To set up {metrics} monitoring, you need: + +* An Elasticsearch cluster and Kibana (version 6.5 or later) with a basic license +// Add a link to what constitutes a basic license. And is this any different for the cloud? + +* Appropriate {beats} shippers (version 6.5 or later) installed and enabled on each system you want to +monitor + +If your data uses nonstandard fields, you may also need to modify some default configuration settings. + +[float] +=== Get Elasticsearch and Kibana + +To get started, you can use our hosted {es} Service on Elastic Cloud (recommended for new users), or you can install {es} and {kib} locally. + +[float] +==== Use our hosted service + +The hosted {es} Service is available on both AWS and GCP. +{ess-trial}[Try out the {es} Service for free]. + +[float] +==== Install {es} and {kib} locally + +Alternatively, you can {stack-gs}/get-started-elastic-stack.html[install {es} and {kib} locally]. +Follow the instructions to install {es}, and to install and start {kib}. + +[float] +[[install-beats-for-infra-UI]] +=== Install {beats} shippers + +To start collecting metrics data, you need to install and configure the {metricbeat} {beats} shipper. + +You can install and configure {beats} shippers for most kinds of data directly from {kib}, or you can install {beats} yourself. + +[float] +==== Install {beats} from {kib} + +To install {beats} from {kib}, on the machine where you want to collect the data, open a {kib} browser window. +In the *Add Data to Kibana* section, click *Add metric data*. +Now follow the instructions for the type of data you want to collect. +The instructions walk you through the steps required to download, install and configure the appropriate Beats modules for your data. + +[role="screenshot"] +image::images/add-data.png[] + +[float] +==== Install {beats} yourself + +If your data source doesn't have a {beats} module, or if you want to install {beats} the old fashioned way, follow the instructions in {metricbeat-ref}/metricbeat-getting-started.html[{metricbeat} getting started] and enable modules for the metrics you want to collect. + +[float] +=== Enable modules +However you install {beats}, you need to enable the appropriate modules in {metricbeat} to start collecting metrics data. + +// ++ I think some of this is still necessary even if you've followed the instructions to install Beats from Kibana. +// ++ The instructions there explain how to enable the module. Below, we enable more stuff. +// ++ What about if you are using Cloud? Is anything different? + +To populate the *Hosts* view with metrics data, enable: + +* {metricbeat-ref}/metricbeat-module-system.html[{metricbeat} `system` module] (enabled by default) +* {metricbeat-ref}/add-host-metadata.html[{metricbeat} `add_host_metadata` processor] (enabled by default) +* {metricbeat-ref}/add-cloud-metadata.html[{metricbeat} `add_cloud_metadata` processor] (enabled by default) + +To populate the *Docker* view with metrics data, enable: + +* {metricbeat-ref}/metricbeat-module-docker.html[{metricbeat} `docker` module] +* {metricbeat-ref}/add-docker-metadata.html[{metricbeat} `add_docker_metadata` processor] + +To populate the *Kubernetes* view with metrics data, enable: + +* {metricbeat-ref}/metricbeat-module-kubernetes.html[{metricbeat} `kubernetes` module] +* {metricbeat-ref}/add-kubernetes-metadata.html[{metricbeat} `add_kubernetes_metadata` processor] + +[float] +=== Configure your data sources + +If your metrics data has nonstandard fields, you may need to modify some configuration settings in {kib}, such as the index pattern used to query the data, and the timestamp field used for sorting. +You can use the *Settings* tab in the {kibana-ref}/infra-ui.html[{metrics-app}] to modify some common settings. +Alternatively, see {kibana-ref}/infrastructure-ui-settings-kb.html[{metrics} settings] for a complete list of metrics configuration settings. + +[float] +=== More about container monitoring + +If you're monitoring Docker containers or Kubernetes pods, you can use autodiscover to automatically change the configuration settings in response to changes in your containers. +This ensures you don't stop collecting data when your container configuration changes. +To learn how to do this, see {metricbeat-ref}/configuration-autodiscover.html[{metricbeat} autodiscover configuration] \ No newline at end of file diff --git a/docs/en/metrics/kubernetes-metricset.asciidoc b/docs/en/metrics/kubernetes-metricset.asciidoc new file mode 100644 index 000000000..d253fde8b --- /dev/null +++ b/docs/en/metrics/kubernetes-metricset.asciidoc @@ -0,0 +1,19 @@ +[[kubernetes-metricset]] +[role="xpack"] + +=== Kubernetes Pod Fields + +*ID*:: `kubernetes.pod.uid` +*Name*:: `kubernetes.pod.name` +*IP Address*:: `kubernetes.pod.ip` + +[float] +=== Kubernetes Pod Metrics + +*CPU Usage*:: Average of `kubernetes.pod.cpu.usage.node.pct` + +*Memory Usage*:: Average of `kubernetes.pod.memory.usage.node.pct` + +*Inbound Traffic*:: Derivative of the maximum of `kubernetes.pod.network.rx.bytes` scaled to a 1 second rate + +*Outbound Traffic*:: Derivative of the maximum of `kubernetes.pod.network.tx.bytes` scaled to a 1 second rate diff --git a/docs/en/metrics/metrics-overview.asciidoc b/docs/en/metrics/metrics-overview.asciidoc new file mode 100644 index 000000000..efead40cb --- /dev/null +++ b/docs/en/metrics/metrics-overview.asciidoc @@ -0,0 +1,32 @@ +[[infrastructure-monitoring-overview]] +[role="xpack"] +== Metrics monitoring overview + +The {metrics-app} enables you to monitor metrics for your infrastructure and identify problems in real time. +You can view basic metrics for servers, containers, services, and so on. +Then you can drill down to view more detailed metrics, or you can seamlessly switch to view corresponding logs, uptime information or APM traces where available. +// Add links to logs, uptime and APM when I have good places to link to. + +[float] +=== Metrics monitoring components + +image::images/metrics-monitoring-architecture.png[] + +// redo for metrics and logs separately. + +Metrics monitoring requires the following {stack} components. + +*https://www.elastic.co/products/elasticsearch[{es}]* is a real-time, +distributed storage, search, and analytics engine. {es} can store, search, and analyze large volumes of data in near real-time. +The {metrics-app} uses {es} to store metrics data in {es} documents which are queried on demand. + +*https://www.elastic.co/products/beats[{beats}]* are open source data shippers that you install as agents on your servers to send data to {es}. +The {metrics-app} uses metrics collected by Metricbeat from the servers, containers, and other services in your infrastructure. +Metricbeat modules are available for most common servers, containers and services. + +*https://www.elastic.co/products/kibana[{kib}]* is an open source analytics and visualization platform designed to work with {es}. +You use {kib} to search, view, and interact with the metrics data stored in {es}. +You can easily perform advanced data analysis and visualize your data in a variety of charts, tables, +and maps. +The <> in {kib} provides a dedicated user interface to view metrics for your infrastructure. + diff --git a/docs/en/secops/index.asciidoc b/docs/en/secops/index.asciidoc deleted file mode 100644 index ef6afe363..000000000 --- a/docs/en/secops/index.asciidoc +++ /dev/null @@ -1,17 +0,0 @@ -:doctype: book -:sec: SecOps -:sec-soln: security monitoring -:sec-ui: SecOps - -= Security Monitoring Guide - -include::{asciidoc-dir}/../../shared/versions.asciidoc[] - -include::{asciidoc-dir}/../../shared/attributes.asciidoc[] - -include::overview.asciidoc[] - -include::installation.asciidoc[] - -include::sec-ui.asciidoc[] - diff --git a/docs/en/secops/installation.asciidoc b/docs/en/secops/installation.asciidoc deleted file mode 100644 index 1c1bafe3f..000000000 --- a/docs/en/secops/installation.asciidoc +++ /dev/null @@ -1,42 +0,0 @@ -[[install-sec-monitoring]] -[role="xpack"] -== Get up and running - -beta[] - -To get up and running with security monitoring, you need: - -* An Elasticsearch cluster and Kibana (version 6.x or later) with a basic -license. To learn how to get started quickly, see -{stack-gs}/get-started-elastic-stack.html[Getting started with the {stack}]. -+ -[TIP] -============== -You can skip installing {es} and {kib} by using our -https://www.elastic.co/cloud/elasticsearch-service[hosted {es} Service] on -Elastic Cloud. The {es} Service is available on both AWS and GCP. -https://www.elastic.co/cloud/elasticsearch-service/signup[Try the {es} -Service for free]. -============== - -* {beats} shippers (version 6.x or later) installed on each system you want to -monitor - -You might need to modify UI settings in {kib} to change default behaviors, -such as the index pattern used to query the data, and the timestamp field used -for sorting. For more information, see {kib}. - -[float] -[[install-beats-for-sec]] -=== Install {beats} shippers - -To populate the security UI with metrics and -log data, you need to install and configure the following shippers: - -* https://www.elastic.co/products/beats/packetbeat[{packetbeat}] for analyzing -network packets -* https://www.elastic.co/products/beats/filebeat[{filebeat}] for forwarding and -centralizing logs and files -* https://www.elastic.co/products/beats/auditbeat[{auditbeat}] for monitoring -directories for file changes - diff --git a/docs/en/secops/overview.asciidoc b/docs/en/secops/overview.asciidoc deleted file mode 100644 index 562149c6f..000000000 --- a/docs/en/secops/overview.asciidoc +++ /dev/null @@ -1,20 +0,0 @@ -[[sec-monitoring-overview]] -[role="xpack"] -== Overview - -beta[] - -{sec} gives you a comprehensive view into your security operations. - -The UI in {kib} UI to brings together data -from a variety of sources, making it easier for you -to identify and resolve security issues. - -[float] -[[secops-components]] -=== Security monitoring components - -Security monitoring requires the following {stack} components. - -image::images/secops-architecture.png[] - diff --git a/docs/en/secops/sec-ui.asciidoc b/docs/en/secops/sec-ui.asciidoc deleted file mode 100644 index eb00180a0..000000000 --- a/docs/en/secops/sec-ui.asciidoc +++ /dev/null @@ -1,14 +0,0 @@ -[[sec-ui-overview]] -[role="xpack"] -== {sec-ui} UI - -beta[] - -After you have security monitoring <> and data is streaming to {es}, use the {sec-ui} UI in {kib} to monitor -and identify security problems in real time. - -For more information about working with the {sec-ui} UI, see the -{kib} documentation. - - diff --git a/docs/en/siem/case-api-update-connector.asciidoc b/docs/en/siem/case-api-update-connector.asciidoc new file mode 100644 index 000000000..e782214e0 --- /dev/null +++ b/docs/en/siem/case-api-update-connector.asciidoc @@ -0,0 +1,83 @@ +[[case-api-update-connector]] +=== Update case configurations + +Updates the connector's case closure settings. + +Connectors are used to interface with external systems. You can only call this +method after you have created a connector (see <>). After a +connector has been created and assigned, call <> to +send cases to the external system. + +==== Request URL + +`PATCH :/api/cases/configure` + +==== Request body + +A JSON object with these fields: + +|============================================== +|Name |Type |Description |Required + +|`connector_id` |String |The ID of the connector you want to use for sending +cases to external systems. |No +|`connector_name` |String |The connector name. |No +|`closure_type` |String a|Determines whether a case is automatically closed in +the {siem-app} when it is pushed to {sn}. Valid values are: + +* `close-by-pushing`: {siem-app} cases are automatically closed when they +are pushed to {sn}. +* `close-by-user`: {siem-app} cases are not automatically closed. + +|No + +|`version` |String |The connector version. |Yes +|============================================== + +NOTE: Call <> to retrieve the version value, and +<> to retrieve connector IDs. + +===== Example request + +Changes the connector's case closure option: + +[source,sh] +-------------------------------------------------- +PATCH api/cases/configure +{ + "connector_id": "131d4448-abe0-4789-939d-8ef60680b498", + "closure_type": "close-by-pushing", + "version": "WzIwMiwxXQ==" +} +-------------------------------------------------- +// KIBANA + +==== Response code + +`200`:: + Indicates a successful call. + +===== Example response + +[source,json] +-------------------------------------------------- +{ + "connector_id": "131d4448-abe0-4789-939d-8ef60680b498", + "closure_type": "close-by-pushing", + "connector_name": "ServiceNow", + "created_at": "2020-03-30T13:31:38.083Z", + "created_by": { + "email": "admin@hms.gov.uk", + "full_name": "Ms Admin", + "username": "admin" + }, + "updated_at": "2020-03-31T06:21:35.759Z", + "updated_by": { + "email": "admin@hms.gov.uk", + "full_name": "Ms Admin", + "username": "admin" + }, + "version": "WzIwMywxXQ==" +} +-------------------------------------------------- + diff --git a/docs/en/siem/cases-api-add-comment.asciidoc b/docs/en/siem/cases-api-add-comment.asciidoc new file mode 100644 index 000000000..a4802a3de --- /dev/null +++ b/docs/en/siem/cases-api-add-comment.asciidoc @@ -0,0 +1,98 @@ +[[cases-api-add-comment]] +=== Add comment + +Adds a comment to an existing case. + +==== Request URL + +`POST :/api/cases//comments` + +===== URL parts + +The URL must include the `case ID` of the case to which you are adding a +comment. Call <> to retrieve case IDs. + +==== Request body + +A JSON object with a `comment` field: + +[width="100%",options="header"] +|============================================== +|Name |Type |Description |Required + +|`comment` |String |The case's new comment. |Yes +|============================================== + +===== Example request + +Adds a comment to case ID `293f1bc0-74f6-11ea-b83a-553aecdb28b6`: + +[source,sh] +-------------------------------------------------- +POST api/cases/293f1bc0-74f6-11ea-b83a-553aecdb28b6/comments +{ + "comment": "That is nothing - Ethan Hunt answered a targeted social + media campaign promoting phishy pension schemes to IMF operatives." +} +-------------------------------------------------- +// KIBANA + +==== Response code + +`200`:: + Indicates a successful call. + +==== Response payload + +The updated JSON case object with the new comment, the user who created the +comment, and the comment's ID, version, and creation time. + +===== Example response + +[source,json] +-------------------------------------------------- + + "id": "293f1bc0-74f6-11ea-b83a-553aecdb28b6", + "version": "WzIwNDMyLDFd", + "comments": [ + { + "id": "8af6ac20-74f6-11ea-b83a-553aecdb28b6", + "version": "WzIwNDMxLDFd", + "comment": "That is nothing - Ethan Hunt answered a targeted social media campaign promoting phishy pension schemes to IMF operatives.", + "created_at": "2020-04-02T15:28:03.034Z", + "created_by": { + "email": "moneypenny@hms.gov.uk", + "full_name": "Ms Moneypenny", + "username": "moneypenny" + }, + "pushed_at": null, + "pushed_by": null, + "updated_at": null, + "updated_by": null + } + ], + "totalComment": 0, + "title": "This case will self-destruct in 5 seconds", + "tags": [ + "phishing", + "social engineering" + ], + "description": "James Bond clicked on a highly suspicious email banner advertising cheap holidays for underpaid civil servants.", + "closed_at": null, + "closed_by": null, + "created_at": "2020-04-02T15:25:19.088Z", + "created_by": { + "email": "ahunley@imf.usa.gov", + "full_name": "Alan Hunley", + "username": "ahunley" + }, + "external_service": null, + "status": "open", + "updated_at": "2020-04-02T15:28:03.034Z", + "updated_by": { + "email": "moneypenny@hms.gov.uk", + "full_name": "Ms Moneypenny", + "username": "moneypenny" + } +} +-------------------------------------------------- \ No newline at end of file diff --git a/docs/en/siem/cases-api-assign-connector.asciidoc b/docs/en/siem/cases-api-assign-connector.asciidoc new file mode 100644 index 000000000..7901fd773 --- /dev/null +++ b/docs/en/siem/cases-api-assign-connector.asciidoc @@ -0,0 +1,72 @@ +[[assign-connector]] +=== Set connector + +Sets the default connector in the {siem-ui}. + +Connectors are used to interface with external systems. You can only call this +method after you have created a connector (see <>). After a +connector has been created and assigned, call <> to +send cases to the external system. + +==== Request URL + +`POST :/api/cases/configure` + +==== Request body + +A JSON object with these fields: + +[width="100%",options="header"] +|============================================== +|Name |Type |Description |Required + +|`connector_id` |String |The connector ID. |Yes +|`connector_name` |String |The connector name. |Yes +|`closure_type` |String a|Determines whether a case is automatically closed in +the {siem-app} when it is pushed to {sn}. Valid values are: + +* `close-by-pushing`: {siem-app} cases are automatically closed when they +are pushed to {sn}. +* `close-by-user`: {siem-app} cases are not automatically closed. + +|Yes +|============================================== + +NOTE: Call <> to retrieve connector IDs and names. + +==== Example request + +[source,sh] +-------------------------------------------------- +POST api/cases/configure +{ + "connector_id": "61787f53-4eee-4741-8df6-8fe84fa616f7", + "closure_type": "close-by-user", + "connector_name": "ServiceNow" +} +-------------------------------------------------- + +==== Response code + +`200`:: + Indicates a successful call. + +==== Example response + +[source,json] +-------------------------------------------------- +{ + "connector_id": "61787f53-4eee-4741-8df6-8fe84fa616f7", + "closure_type": "close-by-user", + "connector_name": "ServiceNow", + "created_at": "2020-03-30T13:31:38.083Z", + "created_by": { + "email": "moneypenny@hms.gov.uk", + "full_name": "Ms Moneypenny", + "username": "moneypenny" + }, + "updated_at": null, + "updated_by": null, + "version": "WzE3NywxXQ==" +} +-------------------------------------------------- diff --git a/docs/en/siem/cases-api-associate-sn.asciidoc b/docs/en/siem/cases-api-associate-sn.asciidoc new file mode 100644 index 000000000..3be233faa --- /dev/null +++ b/docs/en/siem/cases-api-associate-sn.asciidoc @@ -0,0 +1,103 @@ +[[cases-api-associate-sn-incident]] +=== Add external details to case + +Adds the data returned from {sn} to the specified case. + +After sending a new or updated case to {sn}, you must associate the returned +{sn} incident fields with the case in the {siem-app}. + +==== Request URL + +`POST :/api/cases//_push` + +===== URL parts + +The URL must include the `case ID` of the case you are updating. Call +<> to retrieve case IDs. + +==== Request body + +A JSON object with the data returned from {sn}: + +[width="100%",options="header"] +|============================================== +|Name |Type |Description |Required + +|`connector_id` |String |The ID of the connector used to send the case to {sn}. +|Yes +|`connector_name` |String |The ID of the connector used to send the case to +{sn}. |Yes +|`external_id` |String |The `incidentId` returned when calling +<>. |Yes +|`external_title` |String |The `number` returned when calling +<>. |Yes +|`external_url` |String |The `url` returned when calling +<>. |Yes +|============================================== + +===== Example request + +[source,sh] +-------------------------------------------------- +POST api/cases/718265d0-733a-11ea-a0b2-c51ea50a58e2/_push + + "connector_id": "61787f53-4eee-4741-8df6-8fe84fa616f7", + "connector_name": "ServiceNow", + "external_id": "74c15d07dbb300106ba884da0b9619a0", + "external_title": "INC0010016", + "external_url": "https://dev78437.service-now.com/nav_to.do?uri=incident.do?sys_id=74c15d07dbb300106ba884da0b9619a0" +} +-------------------------------------------------- +// KIBANA + +==== Response code + +`200`:: + Indicates a successful call. + +==== Response payload + +The updated JSON case object. + +===== Example response + +[source,json] +-------------------------------------------------- +{ + "id": "718265d0-733a-11ea-a0b2-c51ea50a58e2", + "version": "WzMyNywxXQ==", + "comments": [], + "totalComment": 0, + "closed_at": null, + "closed_by": null, + "created_at": "2020-03-31T10:29:03.781Z", + "created_by": { + "email": "ahunley@imf.usa.gov", + "full_name": "Alan Hunley", + "username": "ahunley" + }, + "external_service": { + "pushed_at": "2020-03-31T10:56:10.959Z", + "pushed_by": { + "username": "ahunley", + "full_name": "Alan Hunley", + "email": "ahunley@imf.usa.gov" + }, + "connector_id": "61787f53-4eee-4741-8df6-8fe84fa616f7", + "connector_name": "ServiceNow", + "external_id": "74c15d07dbb300106ba884da0b9619a0", + "external_title": "INC0010016", + "external_url": "https://dev78437.service-now.com/nav_to.do?uri=incident.do?sys_id=74c15d07dbb300106ba884da0b9619a0" + }, + "updated_at": "2020-03-31T10:56:10.959Z", + "updated_by": { + "username": "ahunley", + "full_name": "Alan Hunley", + "email": "ahunley@imf.usa.gov" + }, + "title": "This case will self-destruct in 5 seconds", + "tags": [], + "description": "James Bond clicked on a highly suspicious email banner advertising cheap holidays for underpaid civil servants.", + "status": "open" +} +-------------------------------------------------- \ No newline at end of file diff --git a/docs/en/siem/cases-api-create.asciidoc b/docs/en/siem/cases-api-create.asciidoc new file mode 100644 index 000000000..04ce05810 --- /dev/null +++ b/docs/en/siem/cases-api-create.asciidoc @@ -0,0 +1,82 @@ +[[cases-api-create]] +=== Create case + +Creates a new case. + +==== Request URL + +`POST :/api/cases` + +==== Request body + +A JSON object with these fields: + +[width="100%",options="header"] +|============================================== +|Name |Type |Description |Required + +|`title` |String |The case's title. |Yes +|`description` |String |The case's description. |Yes +|`tags` |String[] |String array containing words and phrases that help +categorize cases. |Yes, can be an empty array. +|============================================== + +===== Example request + +[source,sh] +-------------------------------------------------- +POST api/cases +{ + "description": "James Bond clicked on a highly suspicious email + banner advertising cheap holidays for underpaid civil servants.", + "title": "This case will self-destruct in 5 seconds", + "tags": [ + "phishing", + "social engineering" + ] +} +-------------------------------------------------- +// KIBANA + +==== Response code + +`200`:: + Indicates a successful call. + +==== Response payload + +A JSON object that includes the user who created the case and the case's ID, +version, and creation time. + +===== Example response + +[source,json] +-------------------------------------------------- +{ + "id": "a18b38a0-71b0-11ea-a0b2-c51ea50a58e2", + "version": "Wzc0LDFd", + "comments": [], + "totalComment": 0, + "closed_at": null, + "closed_by": null, + "created_at": "2020-03-29T11:30:02.658Z", + "created_by": { + "email": "ahunley@imf.usa.gov", + "full_name": "Alan Hunley", + "username": "ahunley" + }, + "external_service": null, <1> + "updated_at": null, + "updated_by": null, + "description": "James Bond clicked on a highly suspicious email banner advertising cheap holidays for underpaid civil servants.", + "title": "This case will self-destruct in 5 seconds", + "status": "open", + "tags": [ + "phishing", + "social engineering" + ] +} +-------------------------------------------------- + +<1> The `external_service` object stores information when the case is pushed to +external systems. For more information, see <>. diff --git a/docs/en/siem/cases-api-delete-all-comments.asciidoc b/docs/en/siem/cases-api-delete-all-comments.asciidoc new file mode 100644 index 000000000..f174af702 --- /dev/null +++ b/docs/en/siem/cases-api-delete-all-comments.asciidoc @@ -0,0 +1,28 @@ +[[cases-api-delete-all-comments]] +=== Delete all comments + +Deletes all comments from the specified case. + +==== Request URL + +`DELETE :/api/cases//comments` + +===== URL parts + +The URL must include the `case ID` of the comment’s associated case (call +<> to retrieve case IDs). + +===== Example request + +Deletes all comments from case ID `9c235210-6834-11ea-a78c-6ffb38a34414`: + +[source,console] +-------------------------------------------------- +DELETE api/cases/a18b38a0-71b0-11ea-a0b2-c51ea50a58e2/comments +-------------------------------------------------- +// KIBANA + +==== Response code + +`204`:: + Indicates a successful call. \ No newline at end of file diff --git a/docs/en/siem/cases-api-delete-case.asciidoc b/docs/en/siem/cases-api-delete-case.asciidoc new file mode 100644 index 000000000..908cfcb02 --- /dev/null +++ b/docs/en/siem/cases-api-delete-case.asciidoc @@ -0,0 +1,33 @@ +[[cases-api-delete-case]] +=== Delete case + +Deletes the specified cases and all associated comments. + +==== Request URL + +`DELETE :/api/cases?ids=["",""]` + +===== URL parts + +The URL must include the case IDs of the cases you are deleting. Call +<> to retrieve case IDs. + +NOTE: All non-ascii characters must be encoded. + +===== Example request + +Deletes cases with these IDs: + +* `2e3a54f0-6754-11ea-a1c2-e3a8bc9f7aca` +* `40b9a450-66a0-11ea-be1b-2bd3fef48984` + +[source,console] +-------------------------------------------------- +DELETE api/cases?ids=%5B%222e3a54f0-6754-11ea-a1c2-e3a8bc9f7aca%22%2C%2240b9a450-66a0-11ea-be1b-2bd3fef48984%22%5D +-------------------------------------------------- +// KIBANA + +==== Response code + +`204`:: + Indicates a successful call. \ No newline at end of file diff --git a/docs/en/siem/cases-api-delete-comment.asciidoc b/docs/en/siem/cases-api-delete-comment.asciidoc new file mode 100644 index 000000000..7c63e7602 --- /dev/null +++ b/docs/en/siem/cases-api-delete-comment.asciidoc @@ -0,0 +1,33 @@ +[[cases-api-delete-comment]] +=== Delete comment + +Deletes the specified comment. + +==== Request URL + +`DELETE :/api/cases//comments/` + +===== URL parts + +The URL must include: + +* the `case ID` of the comment’s associated case (call <> +to retrieve case IDs). +* the `comment ID` of the comment you are retrieving (call +<> to retrieve comment IDs). + +===== Example request + +Deletes comment ID `71ec1870-725b-11ea-a0b2-c51ea50a58e2` from case ID +`a18b38a0-71b0-11ea-a0b2-c51ea50a58e2`: + +[source,sh] +-------------------------------------------------- +DELETE api/cases/a18b38a0-71b0-11ea-a0b2-c51ea50a58e2/comments/71ec1870-725b-11ea-a0b2-c51ea50a58e2 +-------------------------------------------------- +// KIBANA + +==== Response code + +`204`:: + Indicates a successful call. \ No newline at end of file diff --git a/docs/en/siem/cases-api-find-cases.asciidoc b/docs/en/siem/cases-api-find-cases.asciidoc new file mode 100644 index 000000000..623af89bb --- /dev/null +++ b/docs/en/siem/cases-api-find-cases.asciidoc @@ -0,0 +1,131 @@ +[[cases-api-find-cases]] +=== Find cases + +Retrieves a paginated subset of cases. By default, the first page is returned +with 20 results per page. + +NOTE: Cases are saved objects. See +{kibana-ref}/saved-objects-api-find.html[Find objects API] for more query +parameters. + +==== Request URL + +`GET :/api/cases/_find` + +===== URL query parameters + +All parameters are optional: + +[width="100%",options="header"] +|============================================== +|Name |Type |Description + +|`page` |Integer |The page number to return. + +|`perPage` |Integer |The number of rules to return per page. + +|`sortField` |String |Determines which field is used to sort the results, +`createdAt` or `updatedAt`. + +|`sortOrder` |String |Determines the sort order, which can be `desc` or `asc`. + +|`status` |String |Filters the returned cases by state, which can be `open` or +`closed`. + +|`tags` |String |Filters the returned cases by tags. + +|`reporters` |String |Filters the returned cases by the reporter's `username`. + +|============================================== + +NOTE: Even though the JSON case object uses `created_at` and `updated_at` +fields, you must use `createdAt` and `updatedAt` fields in the URL +query. + +===== Example request + +Retrieves the first five cases with the `phishing` tag, in ascending order by +last update time. + +[source,sh] +-------------------------------------------------- +GET api/cases/_find?page=1&perPage=5&sortField=updatedAt&sortOrder=asc&tags=phishing +-------------------------------------------------- +// KIBANA + +==== Response code + +`200`:: + Indicates a successful call. + +==== Response payload + +A JSON object listing the retrieved cases. + +===== Response example + +[source,json] +-------------------------------------------------- +{ + "page": 1, + "per_page": 5, + "total": 2, + "cases": [ + { + "id": "abed3a70-71bd-11ea-a0b2-c51ea50a58e2", + "version": "WzExMCwxXQ==", + "comments": [], + "totalComment": 0, + "closed_at": null, + "closed_by": null, + "created_at": "2020-03-29T13:03:23.533Z", + "created_by": { + "email": "rhustler@aol.com", + "full_name": "Rat Hustler", + "username": "rhustler" + }, + "external_service": null, + "updated_at": null, + "updated_by": null, + "title": "The Long Game", + "tags": [ + "windows", + "phishing" + ], + "description": "Windows 95", + "status": "open" + }, + { + "id": "a18b38a0-71b0-11ea-a0b2-c51ea50a58e2", + "version": "Wzk4LDFd", + "comments": [], + "totalComment": 1, + "closed_at": null, + "closed_by": null, + "created_at": "2020-03-29T11:30:02.658Z", + "created_by": { + "email": "ahunley@imf.usa.gov", + "full_name": "Alan Hunley", + "username": "ahunley" + }, + "external_service": null, + "updated_at": "2020-03-29T12:01:50.244Z", + "updated_by": { + "full_name": "Classified", + "email": "classified@hms.oo.gov.uk", + "username": "M" + }, + "description": "James Bond clicked on a highly suspicious email banner advertising cheap holidays for underpaid civil servants. Operation bubblegum is active. Repeat - operation bubblegum is now active!", + "title": "This case will self-destruct in 5 seconds", + "status": "open", + "tags": [ + "phishing", + "social engineering", + "bubblegum" + ] + } + ], + "count_open_cases": 2, + "count_closed_cases": 0 +} +-------------------------------------------------- \ No newline at end of file diff --git a/docs/en/siem/cases-api-find-connectors.asciidoc b/docs/en/siem/cases-api-find-connectors.asciidoc new file mode 100644 index 000000000..419560e89 --- /dev/null +++ b/docs/en/siem/cases-api-find-connectors.asciidoc @@ -0,0 +1,97 @@ +[[cases-api-find-connectors]] +=== Find connectors + +Retrieves a paginated subset of all {sn} connectors. + +NOTE: Only {sn} connectors are returned. For more information on connectors, +see <>. + +==== Request URL + +`GET :/api/cases/configure/connectors/_find` + +===== Example request + +[source,sh] +-------------------------------------------------- +GET api/cases/configure/connectors/_find +-------------------------------------------------- +// KIBANA + +==== Response code + +`200`:: + Indicates a successful call. + +==== Response payload + +A JSON object describing the connectors and their settings. + +===== Example response + +[source,json] +-------------------------------------------------- +{ + "page": 1, + "perPage": 20, + "total": 2, + "data": [ + { + "id": "61787f53-4eee-4741-8df6-8fe84fa616f7", + "actionTypeId": ".servicenow", + "name": "ServiceNow", + "config": { + "casesConfiguration": { + "mapping": [ + { + "actionType": "overwrite", + "source": "title", + "target": "short_description" + }, + { + "actionType": "append", + "source": "description", + "target": "description" + }, + { + "actionType": "append", + "source": "comments", + "target": "comments" + } + ] + }, + "apiUrl": "https://dev78437.service-now.com" + }, + "referencedByCount": 0 + }, + { + "id": "131d4448-abe0-4789-939d-8ef60680b498", + "actionTypeId": ".servicenow", + "name": "Defector", + "config": { + "apiUrl": "https://dev87359.service-now.com", + "casesConfiguration": { + "mapping": [ + { + "source": "title", + "target": "short_description", + "actionType": "append" + }, + { + "source": "description", + "target": "description", + "actionType": "append" + }, + { + "source": "comments", + "target": "comments", + "actionType": "append" + } + ] + } + }, + "referencedByCount": 0 + } + ] +} +-------------------------------------------------- \ No newline at end of file diff --git a/docs/en/siem/cases-api-get-case-activity.asciidoc b/docs/en/siem/cases-api-get-case-activity.asciidoc new file mode 100644 index 000000000..fa6a28a49 --- /dev/null +++ b/docs/en/siem/cases-api-get-case-activity.asciidoc @@ -0,0 +1,111 @@ +[[cases-api-get-case-activity]] +=== Get all case activity + +Returns all user activity for the specified case. + +==== Request URL + +`GET :/api/cases//user_actions` + +===== URL parts + +The URL must include the `case ID` of the case for which you are retrieving +activity. Call <> to retrieve case IDs. + +===== Example request + +Gets all comments for case ID `a18b38a0-71b0-11ea-a0b2-c51ea50a58e2`: + +[source,sh] +-------------------------------------------------- +GET api/cases/a18b38a0-71b0-11ea-a0b2-c51ea50a58e2/user_actions +-------------------------------------------------- +// KIBANA + +==== Response code + +`200`:: + Indicates a successful call. + +==== Response payload + +A JSON array containing all user activity for the specified case. + +===== Response example + +[source,json] +-------------------------------------------------- +[ + { + "action_field": [ + "description", + "status", + "tags", + "title" + ], + "action": "create", + "action_at": "2020-04-02T15:25:19.088Z", + "action_by": { + "email": "ahunley@imf.usa.gov", + "full_name": "Alan Hunley", + "username": "ahunley" + }, + "new_value": "{\"title\":\"This case will self-destruct in 5 seconds\",\"tags\":[\"phishing\",\"social engineering\"],\"description\":\"James Bond clicked on a highly suspicious email banner advertising cheap holidays for underpaid civil servants.\"}", + "old_value": null, + "action_id": "29ce6370-74f6-11ea-b83a-553aecdb28b6", + "case_id": "293f1bc0-74f6-11ea-b83a-553aecdb28b6", + "comment_id": null + }, + { + "action_field": [ + "comment" + ], + "action": "create", + "action_at": "2020-04-02T15:28:03.034Z", + "action_by": { + "email": "moneypenny@hms.gov.uk", + "full_name": "Ms Moneypenny", + "username": "moneypenny" + }, + "new_value": "That is nothing - Ethan Hunt answered a targeted social media campaign promoting phishy pension schemes to IMF operatives.", + "old_value": null, + "action_id": "8b0d6870-74f6-11ea-b83a-553aecdb28b6", + "case_id": "293f1bc0-74f6-11ea-b83a-553aecdb28b6", + "comment_id": "8af6ac20-74f6-11ea-b83a-553aecdb28b6" + }, + { + "action_field": [ + "comment" + ], + "action": "update", + "action_at": "2020-04-02T15:34:01.118Z", + "action_by": { + "email": "jbond@hms.gov.uk", + "full_name": "James Bond", + "username": "_007" + }, + "new_value": "That is nothing - Ethan Hunt answered a targeted social media campaign promoting phishy pension schemes to IMF operatives. Even worse, he likes baked beans.", + "old_value": "That is nothing - Ethan Hunt answered a targeted social media campaign promoting phishy pension schemes to IMF operatives.", + "action_id": "60dafd50-74f7-11ea-b83a-553aecdb28b6", + "case_id": "293f1bc0-74f6-11ea-b83a-553aecdb28b6", + "comment_id": "8af6ac20-74f6-11ea-b83a-553aecdb28b6" + }, + { + "action_field": [ + "comment" + ], + "action": "create", + "action_at": "2020-04-02T17:48:16.293Z", + "action_by": { + "email": "classified@hms.oo.gov.uk", + "full_name": "Classified", + "username": "M" + }, + "new_value": "I'm on it.", + "old_value": null, + "action_id": "223f7bd0-750a-11ea-b83a-553aecdb28b6", + "case_id": "293f1bc0-74f6-11ea-b83a-553aecdb28b6", + "comment_id": "21a844e0-750a-11ea-b83a-553aecdb28b6" + } +] +-------------------------------------------------- \ No newline at end of file diff --git a/docs/en/siem/cases-api-get-case-comments.asciidoc b/docs/en/siem/cases-api-get-case-comments.asciidoc new file mode 100644 index 000000000..806160c79 --- /dev/null +++ b/docs/en/siem/cases-api-get-case-comments.asciidoc @@ -0,0 +1,74 @@ +[[cases-api-get-all-case-comments]] +=== Get all case comments + +Returns all comments for the specified case. + +==== Request URL + +`GET :/api/cases//comments` + +===== URL parts + +The URL must include the `case ID` of the case for which you are retrieving +comments. Call <> to retrieve case IDs. + +===== Example request + +Gets all comments for case ID `a18b38a0-71b0-11ea-a0b2-c51ea50a58e2`: + +[source,sh] +-------------------------------------------------- +GET api/cases/a18b38a0-71b0-11ea-a0b2-c51ea50a58e2/comments +-------------------------------------------------- +// KIBANA + +==== Response code + +`200`:: + Indicates a successful call. + +==== Response payload + +A JSON array containing all comments for the specified case. + +===== Response example + +[source,json] +-------------------------------------------------- +[ + { + "id": "c6f0dca0-71b3-11ea-a0b2-c51ea50a58e2", + "version": "WzEwNSwxXQ==", + "comment": "That is nothing - Ethan Hunt answered a targeted social media campaign promoting phishy pension schemes to IMF operatives. Even worse, he likes baked beans.", + "created_at": "2020-03-29T11:52:33.889Z", + "created_by": { + "email": "moneypenny@hms.gov.uk", + "full_name": "Ms Moneypenny", + "username": "moneypenny" + }, + "pushed_at": null, + "pushed_by": null, + "updated_at": "2020-03-29T12:14:32.997Z", + "updated_by": { + "email": "jbond@hms.gov.uk", + "full_name": "James Bond", + "username": "_007" + } + }, + { + "id": "71ec1870-725b-11ea-a0b2-c51ea50a58e2", + "version": "WzEyOCwxXQ==", + "comment": "Start operation bubblegum immediately! And chew fast!", + "created_at": "2020-03-30T07:52:46.702Z", + "created_by": { + "email": "classified@hms.oo.gov.uk", + "full_name": "Classified", + "username": "M" + }, + "pushed_at": null, + "pushed_by": null, + "updated_at": null, + "updated_by": null + } +] +-------------------------------------------------- \ No newline at end of file diff --git a/docs/en/siem/cases-api-get-case.asciidoc b/docs/en/siem/cases-api-get-case.asciidoc new file mode 100644 index 000000000..f9644c34d --- /dev/null +++ b/docs/en/siem/cases-api-get-case.asciidoc @@ -0,0 +1,77 @@ +[[cases-api-get-case]] +=== Get case + +Returns the specified case. + +==== Request URL + +`GET :/api/cases/` + +===== URL parts + +The URL must include the `case ID` of the case you are retrieving. Call +<> to retrieve case IDs. + +===== URL query parameters + +[width="100%",options="header"] +|============================================== +|Name |Type |Description |Required + +|`includeComments` |Boolean |Determines whether case comments are +returned. |No, defaults to `true`. +|============================================== + +===== Example request + +Returns case ID `a18b38a0-71b0-11ea-a0b2-c51ea50a58e2` without comments: + +[source,sh] +-------------------------------------------------- +GET api/cases/a18b38a0-71b0-11ea-a0b2-c51ea50a58e2?includeComments=false +-------------------------------------------------- +// KIBANA + +==== Response code + +`200`:: + Indicates a successful call. + +==== Response payload + +The requested case JSON object. + +===== Response example + +[source,json] +-------------------------------------------------- +{ + "id": "a18b38a0-71b0-11ea-a0b2-c51ea50a58e2", + "version": "Wzk4LDFd", + "comments": [], + "totalComment": 0, + "closed_at": null, + "closed_by": null, + "created_at": "2020-03-29T11:30:02.658Z", + "created_by": { + "email": "ahunley@imf.usa.gov", + "full_name": "Alan Hunley", + "username": "ahunley" + }, + "external_service": null, + "updated_at": "2020-03-29T12:01:50.244Z", + "updated_by": { + "full_name": "Classified", + "email": "classified@hms.oo.gov.uk", + "username": "M" + }, + "description": "James Bond clicked on a highly suspicious email banner advertising cheap holidays for underpaid civil servants. Operation bubblegum is active. Repeat - operation bubblegum is now active!", + "title": "This case will self-destruct in 5 seconds", + "status": "open", + "tags": [ + "phishing", + "social engineering", + "bubblegum" + ] +} +-------------------------------------------------- \ No newline at end of file diff --git a/docs/en/siem/cases-api-get-comment.asciidoc b/docs/en/siem/cases-api-get-comment.asciidoc new file mode 100644 index 000000000..ce6ff716e --- /dev/null +++ b/docs/en/siem/cases-api-get-comment.asciidoc @@ -0,0 +1,58 @@ +[[cases-api-get-comment]] +=== Get comment + +Gets the specified comment. + +==== Request URL + +`GET :/api/cases//comments/` + +===== URL parts + +The URL must include: + +* the `case ID` of the comment’s associated case (call <> +to retrieve case IDs). +* the `comment ID` of the comment you are retrieving (call +<> to retrieve comment IDs). + +===== Example request + +Retrieves comment ID `71ec1870-725b-11ea-a0b2-c51ea50a58e2` from case ID +`a18b38a0-71b0-11ea-a0b2-c51ea50a58e2`: + +[source,sh] +-------------------------------------------------- +GET api/cases/a18b38a0-71b0-11ea-a0b2-c51ea50a58e2/comments/71ec1870-725b-11ea-a0b2-c51ea50a58e2 +-------------------------------------------------- +// KIBANA + +==== Response code + +`200`:: + Indicates a successful call. + +==== Response payload + +The requested comment JSON object. + +===== Example response + +[source,json] +-------------------------------------------------- +{ + "id": "71ec1870-725b-11ea-a0b2-c51ea50a58e2", + "version": "WzEyOCwxXQ==", + "comment": "Start operation bubblegum immediately! And chew fast!", + "created_at": "2020-03-30T07:52:46.702Z", + "created_by": { + "email": "classified@hms.oo.gov.uk", + "full_name": "Classified", + "username": "M" + }, + "pushed_at": null, + "pushed_by": null, + "updated_at": null, + "updated_by": null +} +-------------------------------------------------- \ No newline at end of file diff --git a/docs/en/siem/cases-api-get-connector.asciidoc b/docs/en/siem/cases-api-get-connector.asciidoc new file mode 100644 index 000000000..5e5de0b39 --- /dev/null +++ b/docs/en/siem/cases-api-get-connector.asciidoc @@ -0,0 +1,43 @@ +[[cases-get-connector]] +=== Get current connector + +Retrieves the currently used connector. + +NOTE: For more information on connectors, see <>. + +==== Request URL + +`GET :/api/cases/configure` + +===== Example request + +[source,sh] +-------------------------------------------------- +GET api/cases/configure +-------------------------------------------------- +// KIBANA + +==== Response code + +`200`:: + Indicates a successful call. + +===== Example response + +[source,json] +-------------------------------------------------- +{ + "connector_id": "61787f53-4eee-4741-8df6-8fe84fa616f7", + "closure_type": "close-by-user", + "connector_name": "ServiceNow", + "created_at": "2020-03-30T13:31:38.083Z", + "created_by": { + "email": "admin@hms.gov.uk", + "full_name": "Mr Admin", + "username": "admin" + }, + "updated_at": null, + "updated_by": null, + "version": "WzE3NywxXQ==" +} +-------------------------------------------------- \ No newline at end of file diff --git a/docs/en/siem/cases-api-get-reporters.asciidoc b/docs/en/siem/cases-api-get-reporters.asciidoc new file mode 100644 index 000000000..660580a07 --- /dev/null +++ b/docs/en/siem/cases-api-get-reporters.asciidoc @@ -0,0 +1,39 @@ +[[cases-api-get-reporters]] +=== Get reporters + +Returns all case reporters (users who opened cases). + +==== Request URL + +`GET :/api/cases/reporters` + +===== Example request + +[source,sh] +-------------------------------------------------- +GET api/cases/reporters +-------------------------------------------------- +// KIBANA + +==== Response code + +`200`:: + Indicates a successful call. + +===== Example response + +[source,json] +-------------------------------------------------- +[ + { + "full_name": "Alan Hunley", + "email": "ahunley@imf.usa.gov", + "username": "ahunley" + }, + { + "full_name": "Rat Hustler", + "email": "jrhustler@aol.com", + "username": "rhustler" + } +] +-------------------------------------------------- \ No newline at end of file diff --git a/docs/en/siem/cases-api-get-status.asciidoc b/docs/en/siem/cases-api-get-status.asciidoc new file mode 100644 index 000000000..04193367b --- /dev/null +++ b/docs/en/siem/cases-api-get-status.asciidoc @@ -0,0 +1,31 @@ +[[cases-api-get-status]] +=== Get status + +Returns the number of open and closed cases. + +==== Request URL + +`GET :/api/cases/status` + +===== Example request + +[source,sh] +-------------------------------------------------- +GET api/cases/status +-------------------------------------------------- +// KIBANA + +==== Response code + +`200`:: + Indicates a successful call. + +===== Example response + +[source,json] +-------------------------------------------------- +{ + "count_open_cases": 27, + "count_closed_cases": 1198 +} +-------------------------------------------------- \ No newline at end of file diff --git a/docs/en/siem/cases-api-get-tags.asciidoc b/docs/en/siem/cases-api-get-tags.asciidoc new file mode 100644 index 000000000..8ba4d8ac7 --- /dev/null +++ b/docs/en/siem/cases-api-get-tags.asciidoc @@ -0,0 +1,35 @@ +[[cases-api-get-tag]] +=== Get tags + +Aggregates and returns all unique tags from all cases. + +==== Request URL + +`GET :/api/cases/tags` + +===== Example request + +Gets all tags for all cases: + +[source,sh] +-------------------------------------------------- +GET api/cases/tags +-------------------------------------------------- +// KIBANA + +==== Response code + +`200`:: + Indicates a successful call. + +===== Example response + +[source,json] +-------------------------------------------------- +[ + "windows", + "phishing", + "social engineering", + "bubblegum" +] +-------------------------------------------------- \ No newline at end of file diff --git a/docs/en/siem/cases-api-update-comment.asciidoc b/docs/en/siem/cases-api-update-comment.asciidoc new file mode 100644 index 000000000..0c7e5376e --- /dev/null +++ b/docs/en/siem/cases-api-update-comment.asciidoc @@ -0,0 +1,111 @@ +[[cases-api-update-comment]] +=== Update comment + +Updates an existing comment. + +==== Request URL + +`PATCH :/api/cases//comments` + +===== URL parts + +The URL must include the `case ID` of the comment's associated case. Call +<> to retrieve case IDs. + +==== Request body + +A JSON object with the updated comment: + +[width="100%",options="header"] +|============================================== +|Name |Type |Description |Required + +|`comment` |String |The updated comment. |Yes +|`id` |String |The comment's ID. |Yes +|`version` |String |The current comment version. |Yes +|============================================== + +NOTE: Call <> to retrieve comment IDs and +version values. + +===== Example request + +Updates comment ID `8af6ac20-74f6-11ea-b83a-553aecdb28b6` (associated with case +ID `293f1bc0-74f6-11ea-b83a-553aecdb28b6`): + +[source,sh] +-------------------------------------------------- +PATCH api/cases/293f1bc0-74f6-11ea-b83a-553aecdb28b6/comments +{ + "id": "8af6ac20-74f6-11ea-b83a-553aecdb28b6", + "comment": "That is nothing - Ethan Hunt answered a targeted social + media campaign promoting phishy pension schemes to IMF operatives. + Even worse, he likes baked beans.", + "version": "Wzk1LDFd" +} +-------------------------------------------------- +// KIBANA + +==== Response code + +`200`:: + Indicates a successful call. + +==== Response payload + +The updated JSON case object with the updated comment, the user who updated the +comment, and the comment's ID, version, and update time. + +===== Example response + +[source,json] +-------------------------------------------------- +{ + "id": "293f1bc0-74f6-11ea-b83a-553aecdb28b6", + "version": "WzIwNjM2LDFd", + "comments": [ + { + "id": "8af6ac20-74f6-11ea-b83a-553aecdb28b6", + "version": "WzIwNjM3LDFd", + "comment": "That is nothing - Ethan Hunt answered a targeted social media campaign promoting phishy pension schemes to IMF operatives.", + "created_at": "2020-04-02T15:28:03.034Z", + "created_by": { + "email": "moneypenny@hms.gov.uk", + "full_name": "Ms Moneypenny", + "username": "moneypenny" + }, + "pushed_at": null, + "pushed_by": null, + "updated_at": "2020-04-02T15:34:01.118Z", + "updated_by": { + "email": "jbond@hms.gov.uk", + "full_name": "James Bond", + "username": "_007" + } + } + ], + "totalComment": 0, + "title": "This case will self-destruct in 5 seconds", + "tags": [ + "phishing", + "social engineering" + ], + "description": "James Bond clicked on a highly suspicious email banner advertising cheap holidays for underpaid civil servants.", + "closed_at": null, + "closed_by": null, + "created_at": "2020-04-02T15:25:19.088Z", + "created_by": { + "email": "ahunley@imf.usa.gov", + "full_name": "Alan Hunley", + "username": "ahunley" + }, + "external_service": null, + "status": "open", + "updated_at": "2020-04-02T15:34:01.118Z", + "updated_by": { + "email": "jbond@hms.gov.uk", + "full_name": "James Bond", + "username": "_007" + } +} +-------------------------------------------------- \ No newline at end of file diff --git a/docs/en/siem/cases-api-update.asciidoc b/docs/en/siem/cases-api-update.asciidoc new file mode 100644 index 000000000..520a679f8 --- /dev/null +++ b/docs/en/siem/cases-api-update.asciidoc @@ -0,0 +1,112 @@ +[[cases-api-update]] +=== Update case + +Updates existing cases. + +==== Request URL + +`PATCH :/api/cases` + +==== Request body + +A JSON array containing one or more case objects with updated field values: + +[width="100%",options="header"] +|============================================== +|Name |Type |Description |Required + +|`cases` |<> |Array containing one or more case objects. |Yes +|============================================== + +[[update-cases-schema]] +===== `cases` schema +|============================================== +|Name |Type |Description |Required + +|`id` |String |The ID of the case being updated. |Yes +|`title` |String |The updated case title. |No +|`description` |String |The updated case description. |No +|`status` |String a|The updated case status, which can be: + +* `open` +* `closed` + +|No +|`tags` |String[] |The updated case tags. |No +|`version` |String |The current case version (returned when calling +<> or <>). |Yes +|============================================== + +===== Example request + +Updates the `description` and `tags` fields of case ID +`a18b38a0-71b0-11ea-a0b2-c51ea50a58e2`: + +[source,sh] +-------------------------------------------------- +PATCH api/cases +{ + "cases": [ + { + "id": "a18b38a0-71b0-11ea-a0b2-c51ea50a58e2", + "description": "James Bond clicked on a highly suspicious email + banner advertising cheap holidays for underpaid civil servants. + Operation bubblegum is active. Repeat - operation bubblegum is + now active!", + "tags": [ + "phishing", + "social engineering", + "bubblegum" + ], + "version": "WzIzLDFd" + } + ] +} +-------------------------------------------------- +// KIBANA + +==== Response code + +`200`:: + Indicates a successful call. + +==== Response payload + +The updated case with a new `version` value. + +===== Example response + +[source,json] +-------------------------------------------------- +[ + { + "id": "a18b38a0-71b0-11ea-a0b2-c51ea50a58e2", + "version": "Wzk4LDFd", + "comments": [], + "totalComment": 0, + "closed_at": null, + "closed_by": null, + "created_at": "2020-03-29T11:30:02.658Z", + "created_by": { + "email": "ahunley@imf.usa.gov", + "full_name": "Alan Hunley", + "username": "ahunley" + }, + "external_service": null, + "updated_at": "2020-03-29T12:01:50.244Z", + "updated_by": { + "email": "classified@hms.oo.gov.uk", + "full_name": "Classified", + "username": "M" + }, + "description": "James Bond clicked on a highly suspicious email banner advertising cheap holidays for underpaid civil servants. Operation bubblegum is active. Repeat - operation bubblegum is now active!", + "title": "This case will self-destruct in 5 seconds", + "status": "open", + "tags": [ + "phishing", + "social engineering", + "bubblegum" + ] + } +] +-------------------------------------------------- \ No newline at end of file diff --git a/docs/en/siem/cases-api.asciidoc b/docs/en/siem/cases-api.asciidoc new file mode 100644 index 000000000..466bf899f --- /dev/null +++ b/docs/en/siem/cases-api.asciidoc @@ -0,0 +1,63 @@ +[[cases-api-overview]] +[role="xpack"] +== Cases API + +You can create, manage, configure, and send cases to external systems with +these APIs: + +* Cases API: Used to open and manage security action items. The API endpoint is +`:/api/cases`, where `` is the host name and +`` is the port number of your Kibana instance. + +* Actions API: Used to send cases to external systems. The API endpoint +is `:/api/actions`. <> +describes how to set up integrations with third-party systems, and +<> describes how to push {siem-app} cases to third +party systems (currently, ServiceNow). + +NOTE: In dev mode, the Kibana server runs behind a proxy which adds a random +path component to its URL. +{kibana-ref}/development-basepath.html[Considerations for basePath] describes +how to work with and disable the random path component. + +[float] +=== {kib} space API calls + +Cases are created and managed per {kib} space. If you are making calls to a +{kib} space *other than* the `Default` space, the space identifier is part of +the endpoint's URL: + +`:/s//api/cases` + +Where `` is the URL identifier for the space. + +NOTE: You can find space URL identifiers on +{kibana-ref}/xpack-spaces.html[*Edit space*] pages (Management -> Spaces -> +) or by calling +{kibana-ref}/spaces-api-get-all.html[`GET /api/spaces/space`]. + +[float] +=== Authentication + +Token-based authentication, using the same username and password used to log in +to the Kibana UI, is required to access the API. + +[float] +=== API calls + +All calls to the API are stateless. Each call must include all the information +{siem-soln} needs to perform the request. + +All requests must contain these headers: + +* `kbn-xsrf: `, where `` can be any alphanumeric string (for +example, `kbn-xsrf: kibana`) +* `Content-Type: application/json` (when the request contains a JSON payload) + +For example, the following call retrieves the first 20 cases: + +[source,sh] +-------------------------------------------------- +curl -X GET ":/api/cases" +-H 'kbn-xsrf: kibana' -u : +-------------------------------------------------- \ No newline at end of file diff --git a/docs/en/siem/cases-kbn-actions-api.asciidoc b/docs/en/siem/cases-kbn-actions-api.asciidoc new file mode 100644 index 000000000..13f812904 --- /dev/null +++ b/docs/en/siem/cases-kbn-actions-api.asciidoc @@ -0,0 +1,485 @@ +[[cases-actions-api-connectors]] +=== Configure and send cases to external interfaces + +You can create {sn} incidents from {siem-soln} cases. This requires creating +a {sn} connector using the {kib} Actions API. Connectors store the information +required to push cases to {sn} via {sn}'s https://developer.servicenow.com/dev.do#!/reference/api/madrid/rest/c_TableAPI[Table API], such as the {sn} URL, +account credentials, and the field mappings required for representing a +{siem-soln} case as a {sn} incident. + +To send cases to {sn} and keep the {siem-ui} updated: + +. <>: Create the connector. +. <> or <>: If required, configure +connector options. +. [[sn-returned-data]]<>: Send the case to {sn}. You +must store the returned data as it is required for updating the +`external_service` object in the {siem-soln} case. +. [[update-case-sn-data]]<>: Update the +{siem-soln} case with the associated {sn} incident data returned in +<>. +. Repeat <> and <> each +time you open or update a case. + + + +[[register-connector]] +==== Create connector + +Creates a {sn} connector, which can then be used to open {sn} incidents from +{siem-soln} cases. + +===== Request URL + +`POST :/api/action` + +===== Request body + +A JSON object with these fields: + +[width="100%",options="header"] +|============================================== +|Name |Type |Description |Required + +|`actionTypeId` |String |Must be: `.servicenow`. |Yes +|`config` |<> |Object containing the action's +configuration. |Yes +|`secrets` |Object a|Object containing the {sn} account credentials used +to create and update incidents: + +* `username` (string): The account username. +* `password` (string): The account password. + +|Yes + +|`name` |String |The registered {sn} connector. |Yes +|============================================== + +[[config-schema]] +*`config` schema* + +[width="100%",options="header"] +|============================================== +|Name |Type |Description |Required + +|`casesConfiguration` |Object a|Contains a `mapping` array, which determines how {siem-soln} case fields are mapped to {sn} incident fields: + +* `source` (string): The name of the {siem-soln} case field, which can be +`title`, `description`, or `comments`. +* `target` (string): The name of the mapped {sn} incident field. For example: `short_description`, `description`, and `comments`. +* `actionType` (string): Determines whether {siem-soln} case updates overwrite +or append to the mapped {sn} incident fields. Valid values are `overwrite` and +`append`. + +|Yes + +|`apiUrl` |String |URL of the {sn} instance. |Yes +|============================================== + +===== Example request + +[source,sh] +-------------------------------------------------- +POST api/action +{ + "actionTypeId": ".servicenow", + "config": { + "casesConfiguration": { + "mapping": [ + { + "source": "title", <1> + "target": "short_description", + "actionType": "overwrite" + }, + { + "source": "description", <2> + "target": "description", + "actionType": "overwrite" + }, + { + "source": "comments", <3> + "target": "comments", + "actionType": "append" + } + ] + }, + "apiUrl": "https://dev87359.service-now.com" + }, + "secrets": { + "username": "admin", + "password": "securePassword123!" + }, + "name": "ServiceNow" +} +-------------------------------------------------- +// KIBANA + +<1> {siem-soln} case `title` fields are mapped to {sn} `short_description` +fields. When a {siem-soln} `title` field is updated and sent to {sn}, the {sn} +`short_description` field is overwritten. + +<2> {siem-soln} case `description` fields are mapped to {sn} `description` +fields. When a {siem-soln} `description` field is updated and sent to {sn}, +the {sn} `description` field is overwritten. + +<3> {siem-soln} case `comments` fields are mapped to {sn} `comments` fields. +When a {siem-soln} `comments` field is updated and sent to {sn}, the updated +text is appended to the {sn} `comments` field. + +===== Response code + +`200`:: + Indicates a successful call. + +===== Response payload + +A JSON object with a connector `id` that is required to push cases to {sn}. + +===== Example response + +[source,json] +-------------------------------------------------- +{ + "id": "61787f53-4eee-4741-8df6-8fe84fa616f7", + "actionTypeId": ".servicenow", + "name": "ServiceNow", + "config": { + "casesConfiguration": { + "mapping": [ + { + "source": "title", + "target": "short_description", + "actionType": "overwrite" + }, + { + "source": "description", + "target": "description", + "actionType": "overwrite" + }, + { + "source": "comments", + "target": "comments", + "actionType": "append" + } + ] + }, + "apiUrl": "https://dev78437.service-now.com" + } +} +-------------------------------------------------- + +[[update-connector]] +==== Update connector + +Updates a {sn} connector. + +===== Request URL + +`PUT :/api/action/` + +===== URL parts + +The URL must include the `connector ID` of the connector you are updating. +Call <> to retrieve connector IDs. + +===== Request body + +A JSON object with the fields you want to update: + +[width="100%",options="header"] +|============================================== +|Name |Type |Description |Required + +|`config` |<> |Object containing the action's +configuration. |Yes +|`secrets` |Object a|Object containing the {sn} account credentials used +to create and update incidents: + +* `username` (string): The account username. +* `password` (string): The account password. + +|Yes + +|`name` |String |The registered {sn} connector. |Yes +|============================================== + +[[config-update-schema]] +*`config` schema* + +[width="100%",options="header"] +|============================================== +|Name |Type |Description |Required + +|`casesConfiguration` |Object a|Contains a `mapping` array, which determines how {siem-soln} case fields are mapped to {sn} incident fields: + +* `source` (string): The name of the {siem-soln} case field, which can be +`title`, `description`, or `comments`. +* `target` (string): The name of the mapped {sn} incident field. For example: `short_description`, `description`, and `comments`. +* `actionType` (string): Determines whether {siem-soln} case updates overwrite +or append to the mapped {sn} incident fields. Valid values are `overwrite` and +`append`. + +|Yes + +|`apiUrl` |String |URL of the {sn} instance. |Yes +|============================================== + +===== Example request + +Updates the `description` field mapping of connector ID +`61787f53-4eee-4741-8df6-8fe84fa616f7`: + +[source,sh] +-------------------------------------------------- +PUT api/action/61787f53-4eee-4741-8df6-8fe84fa616f7 +{ + "name": "ServiceNow", + "config": { + "apiUrl": "https://dev78437.service-now.com", + "casesConfiguration": { + "mapping": [ + { + "source": "title", + "target": "short_description", + "actionType": "overwrite" + }, + { + "source": "description", + "target": "description", + "actionType": "append" + }, + { + "source": "comments", + "target": "comments", + "actionType": "append" + } + ] + } + }, + "secrets": { + "username": "admin", + "password": "me4Vered1!" + } +} +-------------------------------------------------- +// KIBANA + +===== Response code + +`200`:: + Indicates a successful call. + +===== Response payload + +The updated JSON connector object. + +===== Example response + +[source,json] +-------------------------------------------------- +{ + "id": "61787f53-4eee-4741-8df6-8fe84fa616f7", + "actionTypeId": ".servicenow", + "name": "ServiceNow", + "config": { + "apiUrl": "https://dev78437.service-now.com", + "casesConfiguration": { + "mapping": [ + { + "source": "title", + "target": "short_description", + "actionType": "overwrite" + }, + { + "source": "description", + "target": "description", + "actionType": "append" + }, + { + "source": "comments", + "target": "comments", + "actionType": "append" + } + ] + } + } +} +-------------------------------------------------- + +[[cases-actions-api-execute]] +==== Create or update an external incident + +Creates a new or updates an existing {sn} incident from a {siem-soln} case. + +NOTE: You can only send cases to external system after you have +<> a connector. After you have sent the case to +{sn}, you must call <> to update the +{siem-soln} case with the relevant {sn} incident details. + +===== Request URL + +`POST :/api/action//_execute` + +===== URL parts + +The URL must include the ServiceNow connector ID. Call +<> to retrieve the currently used connector ID, or +<> to retrieve all connectors IDs. + +===== Request body + +A JSON object with these fields: + +[width="100%",options="header"] +|============================================== +|Name |Type |Description |Required + +|`params` |<> |Contains the {siem-soln} case details +for which you are opening a {sn} incident. |Yes +|============================================== + +[[case-conf-params]] +*`params` schema* + +|============================================== +|Name |Type |Description |Required + +|`caseId` |String |The case ID. |Yes +|`createdAt` |String |The time the case was created, using ISO 8601 with UTC +notation. For example, `2020-03-31T06:40:21.674Z`. |Yes +|`createdBy` |Object a|The user who created the case: + +* `fullName` (string): The user's full name. +* `username` (string): The user's username. + +|Yes + +|`comments` |Object[] a|Array containing case comments: + +* `commentId` (string, required): The comment ID. +* `comment` (string, required): The comment text. +* `createdAt` (string, required): The time the comment was created, using ISO 8601 with +UTC notation. +* `createdBy` (object, required): The user who created the comment, containing +`fullName` and `username` fields. +* `updatedBy` (object, optional): The user who last updated the comment, +containing `fullName` and `username` fields. + +|No + +|`description` |String |The case description. |No +|`incidentId` |String |The {sn} incident ID. Required when updating an existing +{sn} incident. |No +|`title` |String |The case title. |Yes +|`updatedAt` |String |The time the case was updated, using ISO 8601 with UTC +notation. |No +|`updatedBy` |Object a|The user who last updated the case: + +* `fullName` (string): The user's full name. +* `username` (string): The user's username. + +|No +|============================================== + +NOTE: When updating an existing case, call <> or +<> to retrieve the `incidentId`. In the case JSON +object, the `incidentId` value is stored in the `external_id` field. + +===== Example requests + +Creates a new {sn} incident: + +[source,sh] +-------------------------------------------------- +POST api/action/7349772f-421a-4de3-b8bb-2d9b22ccee30/_execute +{ + "params": { + "caseId": "c1472f70-732a-11ea-a0b2-c51ea50a58e2", + "createdAt": "2020-03-31T08:36:45.661Z", + "createdBy": { + "fullName": "Alan Hunley", + "username": "ahunley" + }, + "comments": [ + { + "commentId": "dda30310-732a-11ea-a0b2-c51ea50a58e2", + "comment": "That is nothing - Ethan Hunt answered a targeted social media campaign promoting phishy pension schemes to IMF operatives.", + "createdAt": "2020-03-31T08:37:33.240Z", + "createdBy": { + "fullName": "Ms Moneypenny", + "username": "moneypenny" + } + } + ], + "description": "James Bond clicked on a highly suspicious email banner advertising cheap holidays for underpaid civil servants. Operation bubblegum is active.", + "title": "This case will self-destruct in 5 seconds" + } +} +-------------------------------------------------- +// KIBANA + +Updates an existing {sn} incident: + +[source,sh] +-------------------------------------------------- +POST api/action/7349772f-421a-4de3-b8bb-2d9b22ccee30/_execute +{ + "params": { + "caseId": "c1472f70-732a-11ea-a0b2-c51ea50a58e2", + "createdAt": "2020-03-31T08:36:45.661Z", + "createdBy": { + "fullName": "Alan Hunley", + "username": "ahunley" + }, + "comments": [ + { + "commentId": "8ef6d660-732f-11ea-a0b2-c51ea50a58e2", + "comment": "That is nothing - Ethan Hunt answered a targeted social media campaign promoting phishy pension schemes to IMF operatives.", + "createdAt": "2020-03-31T09:11:08.736Z", + "createdBy": { + "fullName": "Ms Moneypenny", + "username": "moneypenny" + } + } + ], + "incidentId": "cc6ef44bdb7300106ba884da0b9619cf", + "title": "This case will self-destruct in 5 seconds" + } +} +-------------------------------------------------- +// KIBANA + +===== Response code + +`200`:: + Indicates a successful call. + +===== Response payload + +A JSON object with the {sn} incident number and link to the {sn} incident. + +IMPORTANT: You need the returned information to associate it with the original +{siem-soln} case. To add the {sn} incident details to the {siem-soln} case, +call <>. + +===== Example response + +[source,json] +-------------------------------------------------- +{ + "status": "ok", + "actionId": "61787f53-4eee-4741-8df6-8fe84fa616f7", + "data": { + "number": "INC0010012", + "incidentId": "62dc3c8bdb7300106ba884da0b9619ea", + "pushedDate": "2020-03-31T09:01:33.000Z", + "url": "https://dev78437.service-now.com/nav_to.do?uri=incident.do?sys_id=62dc3c8bdb7300106ba884da0b9619ea", + "comments": [ + { + "commentId": "dda30310-732a-11ea-a0b2-c51ea50a58e2", + "pushedDate": "2020-03-31T09:01:34.000Z" + } + ] + } +} +-------------------------------------------------- \ No newline at end of file diff --git a/docs/en/siem/cases-overview.asciidoc b/docs/en/siem/cases-overview.asciidoc new file mode 100644 index 000000000..6b7c7d63c --- /dev/null +++ b/docs/en/siem/cases-overview.asciidoc @@ -0,0 +1,68 @@ +[[cases-overview]] +[role="xpack"] + += Cases (Beta) + +beta[] + +Cases are used to open and track security issues directly in the {siem-app}. +All cases list the original reporter and all users who contribute to a case +(`participants`). Comments support Markdown syntax, and allow linking to saved +<>. Additionally, you can send cases to external +systems from within the {siem-app} (currently {sn}). <> +describes how to set this up. + +You can create and manage cases via the UI or the <>. + +NOTE: To send cases to {sn}, you need the +https://www.elastic.co/subscriptions[appropriate license]. + +IMPORTANT: To make sure you can view and open cases, see <>. + +[role="screenshot"] +image::images/cases-ui-home.png[] + +[float] +[[cases-ui-open]] +== Open a new case + +Open a new case to keep track of security issues and share their details with colleagues. + +. Go to *SIEM* -> *Cases* -> *Create new case*. +. Give the case a name, and add a description and any relevant tags. ++ +TIP: In the `Description` area, you can use +https://www.markdownguide.org/cheat-sheet[Markdown] syntax and insert a +timeline link (click the icon in the top right corner of the area). + +. When ready, create the case. +. If external connections are configured, you can send the case to {sn}. + +[role="screenshot"] +image::images/cases-ui-open.png[] + +[float] +[[cases-ui-manage]] +== Manage existing cases + +You can search existing cases, and filter them by tags, reporter, and status +(open or closed). + +To view a case, click on its name. You can then: + +* Add a new comment. +* Edit existing comments and the case's description. +* Send updates to {sn} (if external connections are configured). +* Close the case. +* Reopen a closed case. +* Edit tags. +* Refresh the case to retrieve the latest updates. + +[float] +[[case-permisions]] +== Cases prerequisites + +To view cases, you need the {kib} space `Read` privilege for the Saved Objects +Management feature. To create cases and add comments, you need the `All` {kib} +space privilege for the Saved Objects Management feature. For more information, +see {kibana-ref}/xpack-spaces.html#spaces-control-user-access[Feature access based on user privileges]. diff --git a/docs/en/siem/cases-ui-integrations.asciidoc b/docs/en/siem/cases-ui-integrations.asciidoc new file mode 100644 index 000000000..c1c9cdd40 --- /dev/null +++ b/docs/en/siem/cases-ui-integrations.asciidoc @@ -0,0 +1,61 @@ +[[cases-ui-integrations]] +[role="xpack"] +== Configuring external connections + +You can push new cases and case updates to {sn}. To do this, you need to create +a connector, which stores the information required to push cases to {sn} via +{sn}'s https://developer.servicenow.com/dev.do#!/reference/api/madrid/rest/c_TableAPI[Table API]. +After you have created a connector, you can set {siem-soln} cases to +automatically close when they are sent to {sn}. + +NOTE: To create a {sn} connector and send cases to {sn}, you need the +https://www.elastic.co/subscriptions[appropriate license]. + +[float] +=== Create a new connector + +. Go to *SIEM* -> *Cases* -> *Edit external connection*. ++ +[role="screenshot"] +image::images/cases-ui-connector.png[] +. Click `Add new connector option`, and then click {sn}. ++ +[role="screenshot"] +image::images/cases-ui-sn-connector.png[] +. Fill in the following: +* _Connector name_: A name for the connector. +* _URL_: The URL of the {sn} instance to which you want to send cases. +* _Username_: The username of the {sn} account used to access the {sn} +instance. +* _Password_: The password of the {sn} account used to access the {sn} instance. +. To represent a SIEM case as a {sn} incident, these SIEM case fields are +mapped to {sn} incidents fields as follows: +** `Title`: Mapped to the {sn} `Short description` field. When an update to a +SIEM case title is sent to {sn}, the existing {sn} `Short description` field is +overwritten. +** `Description`: Mapped to the {sn} `Description` field. When an update to a +SIEM case description is sent to {sn}, the existing {sn} `Description` field is +overwritten. +** `Comments`: Mapped to the {sn} `Comments` field. When a comment is updated +in a SIEM case, a new comment is added to the {sn} incident. +. Save the connector. + +[float] +=== Close sent cases automatically + +To close cases when they are sent to {sn}, select the +_Automatically close SIEM cases when pushing new incident to third-party_ +option. + +[float] +=== Change and update connectors + +You can create additional connectors, update existing connectors, and change +the connector used to send cases to {sn}. + +. To change the connector used to send cases to {sn}: +.. Go to *SIEM* -> *Cases* -> *Edit external connection*. +.. Select the required connector from the `Incident management system` list. +. To update an existing connector: +.. Click `Update connector`. +.. Update the connector fields as required. \ No newline at end of file diff --git a/docs/en/siem/detection-engine-intro.asciidoc b/docs/en/siem/detection-engine-intro.asciidoc new file mode 100644 index 000000000..13a4ff762 --- /dev/null +++ b/docs/en/siem/detection-engine-intro.asciidoc @@ -0,0 +1,196 @@ +[[detection-engine-overview]] +[role="xpack"] + += Detections (Beta) + +beta[] + +The SIEM Detections feature automatically searches for threats and creates +signals when they are detected. Signal detection rules define the conditions +for creating signals. Additionally, you can use the {kib} +{kibana-ref}/alerting-getting-started.html[Alerting and Actions] +framework to send notifications via other systems, such as email and Slack, +when signals are generated. + +NOTE: To use {kib} Alerting for signal notifications, you need the +https://www.elastic.co/subscriptions[appropriate license]. + +The {siem-app} comes with <> that search for +suspicious activity on your network and hosts. For information on how to +optimize the prebuilt rules, see <>. You can also +<>. + +You can manage detection rules and signals via the UI or the +<>. + +[IMPORTANT] +============== +To make sure you can access Detections and manage rules, see +<>. +============== + +[role="screenshot"] +image::detections-ui.png[] + +[float] +[[det-engine-terminology]] +== Terminology + +Rules:: +Perform background tasks to detect suspicious activity. There are two types of +rules: + +* Query-based rules search indices for documents that match their queries at +scheduled intervals. When a document matches, a signal is created. +* {ml-cap} rules create a signal when a {ml} job discovers an anomaly score +above a rule's defined threshold. + +Signals:: +Always refer to {siem-soln} produced detections. Signals are never received +from external systems. When a rule's conditions are met, the {siem-app} +writes one or more signals to an Elasticsearch `signals` index. ++ +[NOTE] +============== +Signal indices are created for each {kib} space. The naming convention is: +`.siem-signals-`. For the default space, the signals index is named +`.siem-signals-default`. +============== + +Alerts and events:: +Always refer to data the {siem-app} receives from external systems, such as +Elastic Endpoint and Suricata. + +Actions:: +Used to send notifications via other systems when a signal is created, such as +email, Slack, PagerDuty, and Webhook. + +[float] +== Signals and external alerts + +The Detections page displays all signals and alerts. To view signals created +by a rule, you can: + +* Filter for a specific rule in the KQL bar (for example, +`signal.rule.name :"SSH (Secure Shell) from the Internet"`). +* View signals in the *Rule details* page (click +*Manage signal detection rules* -> rule name in the *All rules* table). + +NOTE: KQL autocomplete for `.siem-signals-*` indices is available on the +*Detections* and *Rule details* pages, and in Timeline when either `All events` +or `Signal events` is selected. + +To view alerts from external data shippers, click *External alerts*. + +[float] +=== Open and close signals + +You can close signals to indicate they do not need any further investigation. +By default, the All signals table displays open signals. To view closed +signals, click *Closed signals*. + +To open and close signals, either: + +* Click the *Close/Open signal* icon. +* Select the signals you want to open or close and then click +*Close/Open selected*. + +[float] +[[signals-to-timelines]] +=== Investigate signals in Timeline + +To investigate a signal in Timeline, click the *Investigate in timeline* +icon. + +If the rule that generated the signal uses a timeline template, when you +investigate the signal in Timeline, the following dropzone query values are +replaced with their corresponding signal values: + +* `host.name` +* `host.hostname` +* `host.domain` +* `host.id` +* `host.ip` +* `client.ip` +* `destination.ip` +* `server.ip` +* `source.ip` +* `network.community_id` +* `user.name` +* `process.name` + +*Example* + +The timeline template used in the rule has this dropzone query: +`host.name: "Linux-LiverpoolFC"`. When signals generated by the rule are +investigated in Timeline, the `host.name` value is replaced with the signal's +`host.name` value. If the signal's `host.name` value is `Windows-ArsenalFC`, +the timeline dropzone query is `host.name: "Windows-ArsenalFC"`. + +NOTE: For information on how to add timeline templates to rules, see +<>. + +[float] +[[detections-permissions]] +== Detections configuration and index privilege prerequisites + +If you are using an *on-premises* {stack} deployment: + +* HTTPS must be configured for communication between +{kibana-ref}/configuring-tls.html#configuring-tls-kib-es[{es} and {kib}]. +* In the `elasticsearch.yml` configuration file, set the +`xpack.security.enabled` setting to `true`. For more information, see +{ref}/settings.html[Configuring {es}] and +{ref}/security-settings.html[Security settings in {es}]. +* In the `kibana.yml` {kibana-ref}/settings.html[configuration file], add the +`xpack.encryptedSavedObjects.encryptionKey` setting with any alphanumeric value +of at least 32 characters. For example: ++ +`xpack.encryptedSavedObjects.encryptionKey: 'fhjskloppd678ehkdfdlliverpoolfcr'` + +For *all* deployments (on-premises and hosted): + +* To view signals and detection rules, you must have at least: +** `read` permissions for the `.siem-signals-` index, where +`` is the name of the {kib} space you are using to view Detections +(see {ref}/security-privileges.html#privileges-list-indices[Indices privileges]). +** {kib} space `Read` privileges for the `SIEM` feature (see +{kibana-ref}/xpack-spaces.html#spaces-control-user-access[Feature access based on user privileges]). +* To create and modify detection rules, you must have: +** {kib} space `All` privileges for the `SIEM` feature (see +{kibana-ref}/xpack-spaces.html#spaces-control-user-access[Feature access based on user privileges]). +** Write permissions for the `.siem-signals-` index, such as +`create` `create_doc`, `write`, `index`, and `all` +(see {ref}/security-privileges.html#privileges-list-indices[Indices privileges]). + +[float] +=== Resolve UI error messages + +Depending on your privileges and whether a `.siem-signals-` index +has already been created for the {kib} space, you might see an error message +when you try to open the *Detections* page. + +*`Let’s set up your detection engine`* + +If you see this message, a user with these privileges must visit the +*Detections* page before you can view signals and rules: + +* The `manage` cluster privilege (see {ref}/security-privileges.html[{es} security privileges]). +* The `create_index` index privilege (see {ref}/security-privileges.html[{es} security privileges]). +* {kib} space `All` privileges for the `SIEM` feature (see +{kibana-ref}/xpack-spaces.html#spaces-control-user-access[Feature access based on user privileges]). + +NOTE: For *on-premises* {stack} deployments only, this message may be displayed +when the +<> +setting has not been added to the `kibana.yml` file. + +*`Detection engine permissions required`* + +If you see this message, you do not have the +<> to view the *Detections* page, +and you should contact your {kib} administrator. + +NOTE: For *on-premises* {stack} deployments only, this message may be +displayed when the <> +setting is not enabled in the `elasticsearch.yml` file. \ No newline at end of file diff --git a/docs/en/siem/images/about-rule-pane.png b/docs/en/siem/images/about-rule-pane.png new file mode 100644 index 000000000..f9bc8fdcf Binary files /dev/null and b/docs/en/siem/images/about-rule-pane.png differ diff --git a/docs/en/siem/images/add-data.png b/docs/en/siem/images/add-data.png new file mode 100644 index 000000000..aaeea1bae Binary files /dev/null and b/docs/en/siem/images/add-data.png differ diff --git a/docs/en/siem/images/all-rules.png b/docs/en/siem/images/all-rules.png new file mode 100644 index 000000000..93541c229 Binary files /dev/null and b/docs/en/siem/images/all-rules.png differ diff --git a/docs/en/siem/images/available-action-types.png b/docs/en/siem/images/available-action-types.png new file mode 100644 index 000000000..101ba5e0d Binary files /dev/null and b/docs/en/siem/images/available-action-types.png differ diff --git a/docs/en/siem/images/cases-ui-connector.png b/docs/en/siem/images/cases-ui-connector.png new file mode 100644 index 000000000..ea36f6a87 Binary files /dev/null and b/docs/en/siem/images/cases-ui-connector.png differ diff --git a/docs/en/siem/images/cases-ui-home.png b/docs/en/siem/images/cases-ui-home.png new file mode 100644 index 000000000..b513efb66 Binary files /dev/null and b/docs/en/siem/images/cases-ui-home.png differ diff --git a/docs/en/siem/images/cases-ui-open.png b/docs/en/siem/images/cases-ui-open.png new file mode 100644 index 000000000..5fb7a6cb6 Binary files /dev/null and b/docs/en/siem/images/cases-ui-open.png differ diff --git a/docs/en/siem/images/cases-ui-sn-connector.png b/docs/en/siem/images/cases-ui-sn-connector.png new file mode 100644 index 000000000..fe69b5ec9 Binary files /dev/null and b/docs/en/siem/images/cases-ui-sn-connector.png differ diff --git a/docs/en/siem/images/cloned-job-details.png b/docs/en/siem/images/cloned-job-details.png new file mode 100644 index 000000000..2e4a7ce4c Binary files /dev/null and b/docs/en/siem/images/cloned-job-details.png differ diff --git a/docs/en/siem/images/create-new-rule.png b/docs/en/siem/images/create-new-rule.png new file mode 100644 index 000000000..6abf18e08 Binary files /dev/null and b/docs/en/siem/images/create-new-rule.png differ diff --git a/docs/en/siem/images/detection-engine.png b/docs/en/siem/images/detection-engine.png new file mode 100644 index 000000000..6501c7c45 Binary files /dev/null and b/docs/en/siem/images/detection-engine.png differ diff --git a/docs/en/siem/images/detections-ui.png b/docs/en/siem/images/detections-ui.png new file mode 100644 index 000000000..5f6691ccd Binary files /dev/null and b/docs/en/siem/images/detections-ui.png differ diff --git a/docs/en/siem/images/edit-dect-rule-filter.png b/docs/en/siem/images/edit-dect-rule-filter.png new file mode 100644 index 000000000..c4a1a95ed Binary files /dev/null and b/docs/en/siem/images/edit-dect-rule-filter.png differ diff --git a/docs/en/siem/images/edit-dect-rule-query.png b/docs/en/siem/images/edit-dect-rule-query.png new file mode 100644 index 000000000..5d247501d Binary files /dev/null and b/docs/en/siem/images/edit-dect-rule-query.png differ diff --git a/docs/en/siem/images/edit-dect-rule.png b/docs/en/siem/images/edit-dect-rule.png new file mode 100644 index 000000000..89ec0e4d9 Binary files /dev/null and b/docs/en/siem/images/edit-dect-rule.png differ diff --git a/docs/en/siem/images/events-count.png b/docs/en/siem/images/events-count.png new file mode 100644 index 000000000..ffa1db74a Binary files /dev/null and b/docs/en/siem/images/events-count.png differ diff --git a/docs/en/siem/images/field-ui.png b/docs/en/siem/images/field-ui.png new file mode 100644 index 000000000..c33c07941 Binary files /dev/null and b/docs/en/siem/images/field-ui.png differ diff --git a/docs/en/siem/images/filter-add-item.png b/docs/en/siem/images/filter-add-item.png new file mode 100644 index 000000000..004380ad3 Binary files /dev/null and b/docs/en/siem/images/filter-add-item.png differ diff --git a/docs/en/siem/images/hosts-ui.png b/docs/en/siem/images/hosts-ui.png new file mode 100644 index 000000000..1c1148a8d Binary files /dev/null and b/docs/en/siem/images/hosts-ui.png differ diff --git a/docs/en/siem/images/ml-rule-threshold.png b/docs/en/siem/images/ml-rule-threshold.png new file mode 100644 index 000000000..98b1beeee Binary files /dev/null and b/docs/en/siem/images/ml-rule-threshold.png differ diff --git a/docs/en/siem/images/ml-ui.png b/docs/en/siem/images/ml-ui.png new file mode 100644 index 000000000..c405da2c7 Binary files /dev/null and b/docs/en/siem/images/ml-ui.png differ diff --git a/docs/en/siem/images/network-ui.png b/docs/en/siem/images/network-ui.png new file mode 100644 index 000000000..9fbc3ecd4 Binary files /dev/null and b/docs/en/siem/images/network-ui.png differ diff --git a/docs/en/siem/images/overview-ui.png b/docs/en/siem/images/overview-ui.png new file mode 100644 index 000000000..e21eec870 Binary files /dev/null and b/docs/en/siem/images/overview-ui.png differ diff --git a/docs/en/siem/images/rule-actions.png b/docs/en/siem/images/rule-actions.png new file mode 100644 index 000000000..eda94444e Binary files /dev/null and b/docs/en/siem/images/rule-actions.png differ diff --git a/docs/en/siem/images/rule-query-example.png b/docs/en/siem/images/rule-query-example.png new file mode 100644 index 000000000..98262d75f Binary files /dev/null and b/docs/en/siem/images/rule-query-example.png differ diff --git a/docs/en/siem/images/rule-scope.png b/docs/en/siem/images/rule-scope.png new file mode 100644 index 000000000..5179ea992 Binary files /dev/null and b/docs/en/siem/images/rule-scope.png differ diff --git a/docs/en/siem/images/schedule-rule.png b/docs/en/siem/images/schedule-rule.png new file mode 100644 index 000000000..8c2153dbb Binary files /dev/null and b/docs/en/siem/images/schedule-rule.png differ diff --git a/docs/en/siem/images/selected-action-type.png b/docs/en/siem/images/selected-action-type.png new file mode 100644 index 000000000..cd1a44099 Binary files /dev/null and b/docs/en/siem/images/selected-action-type.png differ diff --git a/docs/en/siem/images/siem-architecture.png b/docs/en/siem/images/siem-architecture.png new file mode 100644 index 000000000..78f0b2c7c Binary files /dev/null and b/docs/en/siem/images/siem-architecture.png differ diff --git a/docs/en/siem/images/siem-click-swipe.png b/docs/en/siem/images/siem-click-swipe.png new file mode 100644 index 000000000..0f937f91b Binary files /dev/null and b/docs/en/siem/images/siem-click-swipe.png differ diff --git a/docs/en/siem/images/siem-field-highlight.png b/docs/en/siem/images/siem-field-highlight.png new file mode 100644 index 000000000..2b7120249 Binary files /dev/null and b/docs/en/siem/images/siem-field-highlight.png differ diff --git a/docs/en/siem/images/start-job-window.png b/docs/en/siem/images/start-job-window.png new file mode 100644 index 000000000..12c63d0be Binary files /dev/null and b/docs/en/siem/images/start-job-window.png differ diff --git a/docs/en/siem/images/timeline-ui.png b/docs/en/siem/images/timeline-ui.png new file mode 100644 index 000000000..b00d6f308 Binary files /dev/null and b/docs/en/siem/images/timeline-ui.png differ diff --git a/docs/en/siem/index-api-overview.asciidoc b/docs/en/siem/index-api-overview.asciidoc new file mode 100644 index 000000000..1368d54b7 --- /dev/null +++ b/docs/en/siem/index-api-overview.asciidoc @@ -0,0 +1,132 @@ +[[index-api-overview]] +=== Index endpoint + +You use the index endpoint to create, get, and delete a signal index in a +{kib} space. + +You can only create a signal index when the user role has `manage` privileges +for both the {ref}/security-privileges.html[{es} cluster and the +`.siem-signals-` index]. + +When you create a signal index, the following +{ref}/getting-started-index-lifecycle-management.html[{ilm} ({ilm-init})] +policy is created for the signal index: +[source,js] +-------------------------------------------------- +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50gb", + "max_age": "30d" + } + } + } + } + } +} +-------------------------------------------------- + +The `policy` and `rollover_alias` use the same name as the signal index. + +NOTE: To make sure administrators can always create indices, use a glob +pattern that matches indices from multiple spaces in the `Indices` field on +the *Create role* page (*Management* -> *Roles* -> *Create role*). For +example, `.siem-signals-*`. + +==== Create index + +Creates a signal index. The naming convention for the index is +`.siem-signals-`. + +===== Request URL + +`POST :/api/detection_engine/index` + +====== Example request + +Creates a signal index in the {kib} `siem` space. + +[source,console] +-------------------------------------------------- +POST s/siem/api/detection_engine/index +-------------------------------------------------- +// KIBANA + +===== Response code + +`200`:: + Indicates a successful call. + +==== Get index + +Gets the signal index name if it exists. + +===== Request URL + +`GET :/api/detection_engine/index` + +====== Example request + +Gets the signal index for the {kib} `siem` space: + +[source,console] +-------------------------------------------------- +GET s/siem/api/detection_engine/index +-------------------------------------------------- +// KIBANA + +===== Response code + +`200`:: + Indicates a successful call. +`404`:: + Indicates no index exists. + +====== Example responses + +Example response when index exists: + +[source,json] +-------------------------------------------------- +{ + "name": ".siem-signals-siem" +} +-------------------------------------------------- + +Example response when no index exists: + +[source,json] +-------------------------------------------------- +{ + "statusCode": 404, + "error": "Not Found", + "message": "index for this space does not exist" +} +-------------------------------------------------- + +==== Delete index + +Deletes the signal index. + +===== Request URL + +`DELETE :/api/detection_engine/index` + +====== Example request + +Deletes the signal index for the {kib} `siem` space: + +[source, js] +-------------------------------------------------- +DELETE s/siem/api/detection_engine/index +-------------------------------------------------- +// KIBANA + +===== Response code + +`200`:: + Indicates a successful call. \ No newline at end of file diff --git a/docs/en/siem/index.asciidoc b/docs/en/siem/index.asciidoc new file mode 100644 index 000000000..8f58f0e99 --- /dev/null +++ b/docs/en/siem/index.asciidoc @@ -0,0 +1,110 @@ +:doctype: book +:siem-soln: SIEM +:siem-app: SIEM app +:siem-ui: SIEM UI +:ml-dir: {docdir}/../stack/ml/ +:sn: ServiceNow + += SIEM Guide + +include::{asciidoc-dir}/../../shared/versions/stack/{source_branch}.asciidoc[] + +include::{asciidoc-dir}/../../shared/attributes.asciidoc[] + +include::overview.asciidoc[] + +include::installation.asciidoc[] + +include::siem-ui.asciidoc[] + +include::machine-learning.asciidoc[] + +include::tune-anomaly-results.asciidoc[] + +include::detection-engine-intro.asciidoc[] + +include::rules-ui-create.asciidoc[] + +include::rules-api-overview.asciidoc[] + +include::rules-api-create.asciidoc[] + +include::rules-api-get.asciidoc[] + +include::rules-api-find.asciidoc[] + +include::rules-api-update.asciidoc[] + +include::rules-api-delete.asciidoc[] + +include::rules-api-bulk-actions.asciidoc[] + +include::index-api-overview.asciidoc[] + +include::tags-api-overview.asciidoc[] + +include::rules-api-import.asciidoc[] + +include::rules-api-export.asciidoc[] + +include::privileges-api-overview.asciidoc[] + +include::signals-api-overview.asciidoc[] + +include::rules-api-prebuilt.asciidoc[] + +include::prebuilt-rules-reference.asciidoc[] + +include::rule-details/rule-desc-index.asciidoc[] + +include::tune-rule-signals.asciidoc[] + +include::prebuilt-rules-changelog.asciidoc[] + +include::cases-overview.asciidoc[] + +include::cases-ui-integrations.asciidoc[] + +include::cases-api.asciidoc[] + +include::cases-api-create.asciidoc[] + +include::cases-api-add-comment.asciidoc[] + +include::cases-api-update.asciidoc[] + +include::cases-api-update-comment.asciidoc[] + +include::cases-api-find-cases.asciidoc[] + +include::cases-api-get-case.asciidoc[] + +include::cases-api-get-case-comments.asciidoc[] + +include::cases-api-get-comment.asciidoc[] + +include::cases-api-get-case-activity.asciidoc[] + +include::cases-api-get-tags.asciidoc[] + +include::cases-api-get-reporters.asciidoc[] + +include::cases-api-get-status.asciidoc[] + +include::cases-api-delete-comment.asciidoc[] + +include::cases-api-delete-all-comments.asciidoc[] + +include::cases-api-delete-case.asciidoc[] + +include::cases-kbn-actions-api.asciidoc[] + +include::cases-api-assign-connector.asciidoc[] + +include::case-api-update-connector.asciidoc[] + +include::cases-api-get-connector.asciidoc[] + +include::cases-api-find-connectors.asciidoc[] + +include::cases-api-associate-sn.asciidoc[] diff --git a/docs/en/siem/installation.asciidoc b/docs/en/siem/installation.asciidoc new file mode 100644 index 000000000..ea836e463 --- /dev/null +++ b/docs/en/siem/installation.asciidoc @@ -0,0 +1,130 @@ +[chapter, role="xpack"] +[[install-siem]] += Get up and running + +To use the {siem-app}, you need an *Elasticsearch* cluster and *Kibana* +(version 7.2 or later) with a basic license. See +{stack-gs}/get-started-elastic-stack.html[Getting started with the {stack}]. + +There are some additional requirements for using the +<> feature. For more information, see +<>. + +[TIP] +============== +You can skip installing {es} and {kib} by using our +https://www.elastic.co/cloud/elasticsearch-service[hosted {es} Service] on +Elastic Cloud. The {es} Service is available on both AWS and GCP. +{ess-trial}[Try the {es} +Service for free]. +============== + +For information on how to perform cross-cluster searches on {siem-soln} +indices, see: + +* {ref}/modules-cross-cluster-search.html[Search across cluster] +(for on-premises {stack} deployments) +* {cloud}/ec-enable-ccs.html[Enable cross-cluster search] (for hosted deployments) + +[float] +[[siem-ingest]] +== Ingest data + +To ingest data, you can use: + +* *{beats}* shippers (version 7.x or later) installed for each system you want +to monitor. + +* *https://www.elastic.co/products/endpoint-security[Elastic Endpoint Security]*, which ships events and security alerts directly to {es}. + +* Third-party collectors configured to ship ECS-compliant data. + +[IMPORTANT] +============== +If you use a third-party collector to ship data to the {siem-app}, you must +map its fields to the {ecs-ref}[Elastic Common Schema (ECS)]. Additionally, +you must add its index to the {siem-soln} {es} indices (*{kib}* -> +*Management* -> *Advanced Settings* -> *`siem:defaultIndex`*). + +{siem-soln} uses the {ecs-ref}/ecs-host.html[`host.name`] ECS field as the +primary key for identifying hosts. +============== + +[float] +[[install-beats]] +=== Install {beats} shippers + +To populate the {siem-app} with hosts and network security events, you need to install and +configure Beats on the systems from which you want to ingest security events: + +* https://www.elastic.co/products/beats/filebeat[{filebeat}] for forwarding and +centralizing logs and files +* https://www.elastic.co/products/beats/auditbeat[{auditbeat}] for collecting security events +* https://www.elastic.co/products/beats/winlogbeat[{winlogbeat}] for centralizing +Windows event logs +* https://www.elastic.co/products/beats/packetbeat[{packetbeat}] for analyzing +network activity + +You can install {beats} using a {kib}-based guide or directly from the command line. + +[float] +=== Install {beats} using the {kib}-based guide + +Follow the instructions in the Add Data section of the {kib} home page. Click +*Add log data* or *Add metrics*, and follow the links for the types of data you +want to collect. + +[role="screenshot"] +image::add-data.png[] + +[float] +=== Download and install {beats} from the command line + +If your data source isn't in the list, or you want to install {beats} the old +fashioned way: + +* *{filebeat} and {filebeat} modules.* See the +{filebeat-ref}/filebeat-modules-quickstart.html[{filebeat} modules quick start] +and enable modules for the events you want to collect. If there is no module +for the events you want to collect, see the +{filebeat-ref}/filebeat-getting-started.html[{filebeat} getting started] to +learn how to configure inputs. + +* *Auditbeat.* See {auditbeat-ref}/auditbeat-getting-started.html[{auditbeat} getting started]. + +* *Winlogbeat.* See {winlogbeat-ref}/winlogbeat-getting-started.html[{winlogbeat} getting started]. + +* *Packetbeat.* See {packetbeat-ref}/packetbeat-getting-started.html[{packetbeat} getting started]. + +[float] +=== Enable modules and configuration options + +No matter how you installed {beats}, you need to enable modules in {auditbeat} +and {filebeat} to populate the {SIEM-app} with data. + +To populate *Hosts* data, enable these {auditbeat} modules: + +* {auditbeat-ref}/auditbeat-module-system.html[System module - Linux, macOS, Win] +* {auditbeat-ref}/auditbeat-module-auditd.html[Auditd module (Linux Kernel Audit info)] +* {auditbeat-ref}/auditbeat-module-file_integrity.html[File integrity module (FIM) - Linux, macOS, Win] + + +To populate *Network* data, enable the relevant {packetbeat} protocols +and {filebeat} modules: + +* https://www.elastic.co/products/beats/packetbeat[{packetbeat}]: +** {packetbeat-ref}/packetbeat-dns-options.html[DNS] +** {packetbeat-ref}/configuration-tls.html[TLS] +** {packetbeat-ref}/configuration-protocols.html[Other supported protocols] +* https://www.elastic.co/products/beats/filebeat[{filebeat}]: +** {filebeat-ref}/filebeat-module-zeek.html[Zeek NMS module] +** {filebeat-ref}/filebeat-module-suricata.html[Suricata IDS module] +** {filebeat-ref}/filebeat-module-iptables.html[Iptables/Ubiquiti module] +** {filebeat-ref}/filebeat-module-coredns.html[CoreDNS module] +** {filebeat-ref}/filebeat-module-envoyproxy.html[Envoy proxy module (Kubernetes)] +** {filebeat-ref}/filebeat-module-panw.html[Palo Alto Networks firewall module] +** {filebeat-ref}/filebeat-module-cisco.html[Cisco ASA firewall module] +** {filebeat-ref}/filebeat-module-aws.html[AWS module] +** {filebeat-ref}/filebeat-module-cef.html[CEF module] +** {filebeat-ref}/filebeat-module-googlecloud.html[Google Cloud module] +** {filebeat-ref}/filebeat-module-netflow.html[NetFlow module] \ No newline at end of file diff --git a/docs/en/siem/machine-learning.asciidoc b/docs/en/siem/machine-learning.asciidoc new file mode 100644 index 000000000..9250c516a --- /dev/null +++ b/docs/en/siem/machine-learning.asciidoc @@ -0,0 +1,78 @@ +[[machine-learning]] +[role="xpack"] += Anomaly Detection with Machine Learning + +{kibana-ref}/xpack-ml.html[{ml-cap}] functionality is available when +you have the *https://www.elastic.co/subscriptions[appropriate license]*, are +using a *{ess-trial}[cloud deployment]*, or are testing out a *Free Trial*. + +You can view the details of detected anomalies within the `Anomalies` table +widget shown on the Hosts, Network and associated Details pages, or even narrow +to the specific date range of an anomaly from the `Max Anomaly Score` details +in the overview of the Host and IP Details pages. Each of these interfaces also +offer the ability to drag and drop details of the anomaly to Timeline, such as +the `Entity` itself, or any of the associated `Influencers`. + +[role="screenshot"] +image::ml-ui.png[] + + +[float] +[[manage-jobs]] +== Manage machine learning jobs +For users with the `machine_learning_admin` role, the `ML job settings` +interface on the *Detections* page can be used for for viewing, starting, and +stopping SIEM machine learning jobs. + +TIP: To add a custom job to the `ML job settings` interface, add `SIEM` to +the job's `Groups` field ({kib} -> {ml-cap} -> Create/Edit job -> Job +details). + +[float] +[[included-jobs]] +=== Prebuilt jobs + +The SIEM app comes with prebuilt {ml} {anomaly-jobs} for automatically detecting +host and network anomalies. The jobs are displayed in the `Anomaly Detection` +interface. They are available when either: + +* You ship data using https://www.elastic.co/products/beats[Beats], and +{kib} is configured with the required index patterns +(`auditbeat-*`, `filebeat-*`, `packetbeat-*`, or `winlogbeat-*` via {kib} -> Management -> Index Patterns). + +Or + +* Your shipped data is ECS-compliant, and {kib} is configured with the shipped +data's index patterns. + +<> describes all available {ml} jobs, and lists +which ECS fields are required on your hosts when you are not using {beats} to +ship your data. For information on tuning anomaly results to reduce the number +of false positive, see <>. + +NOTE: Machine learning jobs look back and analyse two weeks of historical data +prior to the time they are enabled. After jobs are enabled, they continuously +analyse incoming data. When jobs are stopped and restarted within the two week +timeframe, previously analysed data is not processed again. + +[float] +[[view-anomolies]] +== View detected anomalies +To view the `Anomalies` table widget and `Max Anomaly Score By Job` details, +the user must have the `ml_admin` or `ml_user` role. + +NOTE: To adjust the `score` threshold that determines which +{ml-docs}/xpack-ml.html[anomalies] are shown, you can modify {kib} -> +Management -> Advanced Settings -> `siem:defaultAnomalyScore`. + +[[prebuilt-ml-jobs]] +== Prebuilt job reference + +Prebuilt jobs automatically detect file system and network anomalies on your +hosts. To enable jobs when you are not using {beats}, you must map your data to +the ECS fields listed in each job description. + +NOTE: Some jobs use fields that are not ECS-compliant. These jobs are only +available when you use {beats} to ship data. + +include::{ml-dir}/anomaly-detection/ootb-ml-jobs.asciidoc[tag=siem-jobs] diff --git a/docs/en/siem/overview.asciidoc b/docs/en/siem/overview.asciidoc new file mode 100644 index 000000000..4b09841d3 --- /dev/null +++ b/docs/en/siem/overview.asciidoc @@ -0,0 +1,134 @@ +[[siem-overview]] +[chapter, role="xpack"] += Overview + +{siem-soln} enables analysis of host-related and network-related security events +as part of alert investigations or interactive threat hunting. + +The {siem-app} in {kib} provides an interactive workspace for security teams to +triage events and perform initial investigations. Additionally, +<> and +<> provide ways to +automatically detect suspicious activity across your entire fleet of servers +and workstations. + +[float] +[[siem-components]] +== SIEM components + +SIEM requires the following {stack} components. + +image::images/siem-architecture.png[] + +*https://www.elastic.co/products/endpoint-security[Elastic Endpoint Security]* +is an endpoint security platform and agent that provides prevention, detection, +and response capabilities. It ships events and security alerts directly to {es}. + +*https://www.elastic.co/products/beats[{beats}]* are open source data shippers +that you install as agents on your systems. {beats} send security events and other +data to {es}. + +*https://www.elastic.co/products/elasticsearch[{es}]* is a real-time, +distributed storage, search, and analytics engine. {es} excels at indexing +streams of semi-structured data, such as logs or metrics. + +*https://www.elastic.co/products/kibana[{kib}]* is an open source analytics and +visualization platform designed to work with {es}. You use {kib} to search, +view, and interact with data stored in {es} indices. You can easily perform +advanced data analysis and visualize your data in a variety of charts, tables, +and maps. + +[discrete] +[[siem-integration]] +==== Additional Elastic components + +You can use {siem-soln} with other Elastic products and features to help you +identify and investigate suspicious activity: + +* https://www.elastic.co/products/stack/machine-learning[{ml-cap}] +* https://www.elastic.co/products/stack/alerting[Alerting] +* https://www.elastic.co/products/stack/canvas[Canvas] + +[float] +[[data-sources]] +== Data sources + +SIEM can ingest and analyze data from a variety of sources, including Elastic +Endpoint Security, Beats and Beats modules, +{apm-app-ref}/transactions.html[APM transactions], and third-party +collectors mapped to the {ecs-ref}[Elastic Common Schema (ECS)]. + +[discrete] +[[hosts-data-sources]] +==== Hosts data sources + +* https://www.elastic.co/products/beats/auditbeat[{auditbeat}] +** {auditbeat-ref}/auditbeat-module-system.html[System module - Linux, macOS, Win] +*** packages +*** processes +*** logins +*** sockets +*** users and groups +** {auditbeat-ref}/auditbeat-module-auditd.html[Auditd module (Linux Kernel Audit info)] +** {auditbeat-ref}/auditbeat-module-file_integrity.html[File integrity module (FIM) - Linux, macOS, Win] +* https://www.elastic.co/products/beats/filebeat[{filebeat}] +** system logs (auth logs) - Linux +** Santa - macOS +* https://www.elastic.co/products/beats/winlogbeat[{winlogbeat}] +** Windows event logs - Windows + +[discrete] +[[network-data-sources]] +==== Network data sources + +* https://www.elastic.co/products/beats/packetbeat[{packetbeat}] +** Flows +** DNS +** other protocols +* https://www.elastic.co/products/beats/filebeat[{filebeat}] +** {filebeat-ref}/filebeat-module-zeek.html[Zeek NMS module] +** {filebeat-ref}/filebeat-module-suricata.html[Suricata IDS module] +** {filebeat-ref}/filebeat-module-iptables.html[Iptables/Ubiquiti module] +** {filebeat-ref}/filebeat-module-coredns.html[CoreDNS module] +** {filebeat-ref}/filebeat-module-envoyproxy.html[Envoy proxy module (Kubernetes)] +** {filebeat-ref}/filebeat-module-panw.html[Palo Alto Networks firewall module] +** {filebeat-ref}//filebeat-module-cisco.html[Cisco ASA firewall module] +** {filebeat-ref}/filebeat-module-aws.html[AWS module] +** {filebeat-ref}/filebeat-module-cef.html[CEF module] +** {filebeat-ref}/filebeat-module-googlecloud.html[Google Cloud module] +** {filebeat-ref}/filebeat-module-netflow.html[NetFlow module] + +[discrete] +[[endpoint-security-platform]] +==== Elastic Endpoint Sensor Management Platform + +The Sensor Management Platform (SMP) ships all relevant host and network data +directly to the {siem-app}. + +[discrete] +[[apm-transactions-data]] +==== APM transactions + +By default, the {siem-app} monitors {apm-app-ref}/apm-getting-started.html[APM] +`apm-*-transaction*` indices. To add additional APM indices, update the +index patterns in the `siem:defaultIndex` setting ({kib} -> Management -> +Advanced Settings -> `siem:defaultIndex`). + +[float] +[[ecs]] +=== Elastic Common Schema (ECS) for normalizing data + +The {ecs-ref}[Elastic Common Schema (ECS)] defines a common set of fields to be used for +storing event data in Elasticsearch. ECS helps users normalize their event data +to better analyze, visualize, and correlate the data represented in their +events. + +{siem-soln} can ingest and normalize events from ECS-compatible data sources. + +[discrete] +[[host_id]] +==== Host identification +All Beats use the `add_host_metadata` processor to add the `host.name` field to +events. The default value is `host.name`, but you can change it in Beats +processor settings. + diff --git a/docs/en/siem/prebuilt-rules-changelog.asciidoc b/docs/en/siem/prebuilt-rules-changelog.asciidoc new file mode 100644 index 000000000..34ef58057 --- /dev/null +++ b/docs/en/siem/prebuilt-rules-changelog.asciidoc @@ -0,0 +1,210 @@ +[[prebuilt-rules-changelog]] +== Prebuilt rule changes per release + +beta[] + +The following lists prebuilt rule updates per release. Only rules with +significant modifications to their query or scope are listed. For detailed +information about a rule's changes, see the rule's description page. + +[float] +=== 7.7.0 + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +[float] +=== 7.6.2 + +<> + +[float] +=== 7.6.1 + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + +<> + diff --git a/docs/en/siem/prebuilt-rules-reference.asciidoc b/docs/en/siem/prebuilt-rules-reference.asciidoc new file mode 100644 index 000000000..e437d99f8 --- /dev/null +++ b/docs/en/siem/prebuilt-rules-reference.asciidoc @@ -0,0 +1,279 @@ +[[prebuilt-rules]] +[role="xpack"] +== Prebuilt rule reference + +beta[] + +This section lists all available prebuilt rules. + +IMPORTANT: To run {ml} prebuilt rules, you must have the +https://www.elastic.co/subscriptions[appropriate license] or use a +{ess-trial}[cloud deployment]. All machine learning prebuilt rules are tagged +with `ML`, and their rule type is `machine_learning`. + +[width="100%",options="header"] +|============================================== +|Rule |Description |Tags |Last updated |Version + + +|<