From 12e103ccaaabe407cc8984595b0c84bcad83ddd3 Mon Sep 17 00:00:00 2001 From: Juan A Date: Mon, 5 Feb 2018 10:49:30 +0100 Subject: [PATCH 1/3] document and git-ignore the new keystore provided by beats --- .gitignore | 2 +- docs/copied-from-beats/keystore.asciidoc | 116 +++++++++++++++++++++++ docs/setting-up-and-running.asciidoc | 2 + 3 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 docs/copied-from-beats/keystore.asciidoc diff --git a/.gitignore b/.gitignore index 7ac68950c6d..53860391421 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ .DS_Store *.pyc /apm-server.iml - +/apm-server.keystore /approvals /apm-server /apm-server.test diff --git a/docs/copied-from-beats/keystore.asciidoc b/docs/copied-from-beats/keystore.asciidoc new file mode 100644 index 00000000000..faab29222a8 --- /dev/null +++ b/docs/copied-from-beats/keystore.asciidoc @@ -0,0 +1,116 @@ +////////////////////////////////////////////////////////////////////////// +//// This content is shared by all Elastic Beats. Make sure you keep the +//// descriptions here generic enough to work for all Beats that include +//// this file. When using cross references, make sure that the cross +//// references resolve correctly for any files that include this one. +//// Use the appropriate variables defined in the index.asciidoc file to +//// resolve Beat names: beatname_uc and beatname_lc +//// Use the following include to pull this content into a doc file: +//// include::../../libbeat/docs/keystore.asciidoc[] +////////////////////////////////////////////////////////////////////////// + +[[keystore]] +=== Secrets keystore + +When you configure {beatname_uc}, you might need to specify sensitive settings, +such as passwords. Relying on the file system to protect these values is not +sufficient. {beatname_uc} provides a keystore for storing secret values to use +in configuration settings. + +Unlike the Elasticsearch keystore, the {beatname_uc} keystore does not store +actual configuration settings. Instead, you add a key and secret value to the +keystore, and then use the key in place of the secret value when you configure +sensitive settings. + +The syntax for referencing keys is identical to the syntax for environment +variables: + +`${KEY}` + +Where KEY is the name of the key. + +For example, imagine that the keystore contains a key called `ES_PWD` with the +value `yourelasticsearchpassword`: + +* In the configuration file, use `output.elasticsearch.password: "${ES_PWD}"` +* On the command line, use: `-E "output.elasticsearch.password=${ES_PWD}"` + +When {beatname_uc} unpacks the configuration, it resolves keys before resolving +environment variables and other variables. + +To create and manage keys, use the `keystore` command. See the +<> for the full command syntax, including +optional flags. + +NOTE: The `keystore` command must be run by the same user who will run +{beatname_uc}. + +[float] +[[creating-keystore]] +=== Create a keystore + +To create a secrets keystore, use: + +["source","sh",subs="attributes"] +---------------------------------------------------------------- +{beatname_lc} keystore create +---------------------------------------------------------------- + + +{beatname_uc} creates the keystore in the directory defined by the `path.config` +configuration setting. + +[float] +[[add-keys-to-keystore]] +=== Add keys + +To store sensitive values, such as authentication credentials for Elasticsearch, +use the `keystore add` command: + +["source","sh",subs="attributes"] +---------------------------------------------------------------- +{beatname_lc} keystore add ES_PWD +---------------------------------------------------------------- + + +When prompted, enter a value for the key. + +To overwrite an existing key's value, use the `--force` flag: + +["source","sh",subs="attributes"] +---------------------------------------------------------------- +{beatname_lc} keystore add ES_PWD --force +---------------------------------------------------------------- + +To pass the value through stdin, use the `--stdin` flag. You can also use +`--force`: + +["source","sh",subs="attributes"] +---------------------------------------------------------------- +cat /file/containing/setting/value | {beatname_lc} keystore add ES_PWD --stdin --force +---------------------------------------------------------------- + + +[float] +[[list-settings]] +=== List keys + +To list the keys defined in the keystore, use: + +["source","sh",subs="attributes"] +---------------------------------------------------------------- +{beatname_lc} keystore list +---------------------------------------------------------------- + + +[float] +[[remove-settings]] +=== Remove keys + +To remove a key from the keystore, use: + +["source","sh",subs="attributes"] +---------------------------------------------------------------- +{beatname_lc} remove ES_PWD +---------------------------------------------------------------- + diff --git a/docs/setting-up-and-running.asciidoc b/docs/setting-up-and-running.asciidoc index e9caacbb35a..6ca2a7d6291 100644 --- a/docs/setting-up-and-running.asciidoc +++ b/docs/setting-up-and-running.asciidoc @@ -86,6 +86,8 @@ include::./security.asciidoc[] include::./dashboards.asciidoc[] +include::./copied-from-beats/keystore.asciidoc[] + include::./copied-from-beats/command-reference.asciidoc[] include::./copied-from-beats/shared-directory-layout.asciidoc[] From 6aa12f14bae22ac841bbeb28688ffd66bc413e3a Mon Sep 17 00:00:00 2001 From: Juan A Date: Mon, 5 Feb 2018 13:00:49 +0100 Subject: [PATCH 2/3] update command reference as well --- .../command-reference.asciidoc | 82 +++++++++++++++---- 1 file changed, 68 insertions(+), 14 deletions(-) diff --git a/docs/copied-from-beats/command-reference.asciidoc b/docs/copied-from-beats/command-reference.asciidoc index 1dd132deb78..a8e4cda293d 100644 --- a/docs/copied-from-beats/command-reference.asciidoc +++ b/docs/copied-from-beats/command-reference.asciidoc @@ -16,23 +16,22 @@ :export-command-short-desc: Exports the configuration or index template to stdout :help-command-short-desc: Shows help for any command +:keystore-command-short-desc: Manages the <> :modules-command-short-desc: Manages configured modules :run-command-short-desc: Runs {beatname_uc}. This command is used by default if you start {beatname_uc} without specifying a command - -ifeval::["{has_ml_jobs}"=="yes"] :setup-command-short-desc: Sets up the initial environment, including the index template, Kibana dashboards (when available), and machine learning jobs (when available) -else::[] -:setup-command-short-desc: Sets up the initial environment, including the index template, Kibana dashboards (when available) -endif::[] - :test-command-short-desc: Tests the configuration :version-command-short-desc: Shows information about the current version [[command-line-options]] -=== {beatname_uc} commands +=== {beatname_uc} command reference -{beatname_uc} provides a command-line interface for starting {beatname_uc} and +++++ +Command reference +++++ + +{beatname_uc} provides a command-line interface for running the Beat and performing common tasks, like testing configuration files and loading dashboards. The command-line also supports <> for controlling global behaviors. @@ -57,6 +56,9 @@ endif::[] <>:: {help-command-short-desc}. +<>:: +{keystore-command-short-desc}. + ifeval::[("{beatname_lc}"=="filebeat") or ("{beatname_lc}"=="metricbeat")] <>:: @@ -159,6 +161,62 @@ Specifies the name of the command to show help for. {beatname_lc} help export ----- +[[keystore-command]] +==== `keystore` command + +{keystore-command-short-desc}. + +*SYNOPSIS* + +["source","sh",subs="attributes"] +---- +{beatname_lc} keystore SUBCOMMAND [FLAGS] +---- + +*SUBCOMMANDS* + +*`add KEY`*:: +Adds the specified key to the keystore. Use the `--force` flag to overwrite an +existing key. Use the `--stdin` flag to pass the value through `stdin`. + +*`create`*:: +Creates a keystore to hold secrets. Use the `--force` flag to overwrite the +existing keystore. + +*`list`*:: +Lists the keys in the keystore. + +*`remove KEY`*:: +Removes the specified key from the keystore. + +*FLAGS* + +*`--force`*:: +Valid with the `add` and `create` subcommands. When used with `add`, overwrites +the specified key. When used with `create`, overwrites the keystore. + +*`--stdin`*:: +When used with `add`, uses the stdin as the source of the key's value. + +*`-h, --help`*:: +Shows help for the `keystore` command. + + +{global-flags} + +*EXAMPLES* + +["source","sh",subs="attributes"] +----- +{beatname_lc} keystore create +{beatname_lc} keystore add ES_PWD +{beatname_lc} keystore remove ES_PWD +{beatname_lc} keystore list +----- + +see <> for more examples. + + ifeval::[("{beatname_lc}"=="filebeat") or ("{beatname_lc}"=="metricbeat")] [[modules-command]] @@ -384,10 +442,8 @@ Or: * The index template ensures that fields are mapped correctly in Elasticsearch. * The Kibana dashboards make it easier for you to visualize {beatname_uc} data in Kibana. -ifeval::["{has_ml_jobs}"=="yes"] * The machine learning jobs contain the configuration information and metadata necessary to analyze data for anomalies. -endif::[] Use this command instead of `run --setup` when you want to set up the environment without actually running {beatname_uc} and ingesting data. @@ -408,10 +464,8 @@ Sets up the Kibana dashboards only. *`-h, --help`*:: Shows help for the `setup` command. -ifeval::["{has_ml_jobs}"=="yes"] *`--machine-learning`*:: Sets up machine learning job configurations only. -endif::[] ifeval::["{beatname_lc}"=="filebeat"] @@ -534,7 +588,7 @@ For example: + ["source","sh",subs="attributes"] ---------------------------------------------------------------------- -{beatname_lc} -E "name=mybeat" -E "output.elasticsearch.hosts=["http://myhost:9200"]" +{beatname_lc} -E "name=mybeat" -E "output.elasticsearch.hosts=['http://myhost:9200']" ---------------------------------------------------------------------- + This setting is applied to the currently running {beatname_uc} process. @@ -547,7 +601,7 @@ ifeval::["{beatname_lc}"=="filebeat"] + ["source","sh",subs="attributes"] ---------------------------------------------------------------------- -{beatname_lc} -modules=nginx -M "nginx.access.var.paths=[/var/log/nginx/access.log*]" -M "nginx.access.var.pipeline=no_plugins" +{beatname_lc} -modules=nginx -M "nginx.access.var.paths=['/var/log/nginx/access.log*']" -M "nginx.access.var.pipeline=no_plugins" ---------------------------------------------------------------------- endif::[] From 81a2b44f63042181eeae327fc1dc393469b9a955 Mon Sep 17 00:00:00 2001 From: jalvz Date: Mon, 26 Feb 2018 16:07:45 +0100 Subject: [PATCH 3/3] Incorporate doc updates from https://github.com/elastic/beats/pull/6184 --- .../command-reference.asciidoc | 12 ++++++++-- docs/copied-from-beats/keystore.asciidoc | 23 +++++++++++-------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/docs/copied-from-beats/command-reference.asciidoc b/docs/copied-from-beats/command-reference.asciidoc index a8e4cda293d..02f20350312 100644 --- a/docs/copied-from-beats/command-reference.asciidoc +++ b/docs/copied-from-beats/command-reference.asciidoc @@ -19,7 +19,15 @@ :keystore-command-short-desc: Manages the <> :modules-command-short-desc: Manages configured modules :run-command-short-desc: Runs {beatname_uc}. This command is used by default if you start {beatname_uc} without specifying a command + +ifeval::["{has_ml_jobs}"=="yes"] :setup-command-short-desc: Sets up the initial environment, including the index template, Kibana dashboards (when available), and machine learning jobs (when available) +endif::[] + +ifeval::["{has_ml_jobs}"!="yes"] +:setup-command-short-desc: Sets up the initial environment, including the index template, Kibana dashboards (when available) +endif::[] + :test-command-short-desc: Tests the configuration :version-command-short-desc: Shows information about the current version @@ -31,7 +39,7 @@ Command reference ++++ -{beatname_uc} provides a command-line interface for running the Beat and +{beatname_uc} provides a command-line interface for starting {beatname_uc} and performing common tasks, like testing configuration files and loading dashboards. The command-line also supports <> for controlling global behaviors. @@ -164,7 +172,7 @@ Specifies the name of the command to show help for. [[keystore-command]] ==== `keystore` command -{keystore-command-short-desc}. +{keystore-command-short-desc}. *SYNOPSIS* diff --git a/docs/copied-from-beats/keystore.asciidoc b/docs/copied-from-beats/keystore.asciidoc index faab29222a8..7bdae05603e 100644 --- a/docs/copied-from-beats/keystore.asciidoc +++ b/docs/copied-from-beats/keystore.asciidoc @@ -13,30 +13,33 @@ === Secrets keystore When you configure {beatname_uc}, you might need to specify sensitive settings, -such as passwords. Relying on the file system to protect these values is not -sufficient. {beatname_uc} provides a keystore for storing secret values to use -in configuration settings. +such as passwords. Rather than relying on file system permissions to protect +these values, you can use the {beatname_uc} keystore to securely store secret +values for use in configuration settings. -Unlike the Elasticsearch keystore, the {beatname_uc} keystore does not store -actual configuration settings. Instead, you add a key and secret value to the -keystore, and then use the key in place of the secret value when you configure -sensitive settings. +After adding a key and its secret value to the keystore, you can use the key in +place of the secret value when you configure sensitive settings. The syntax for referencing keys is identical to the syntax for environment variables: `${KEY}` -Where KEY is the name of the key. +Where KEY is the name of the key. For example, imagine that the keystore contains a key called `ES_PWD` with the value `yourelasticsearchpassword`: * In the configuration file, use `output.elasticsearch.password: "${ES_PWD}"` -* On the command line, use: `-E "output.elasticsearch.password=${ES_PWD}"` +* On the command line, use: `-E "output.elasticsearch.password=${ES_PWD}"` When {beatname_uc} unpacks the configuration, it resolves keys before resolving -environment variables and other variables. +environment variables and other variables. + +Notice that the {beatname_uc} keystore differs from the Elasticsearch keystore. +Whereas the Elasticsearch keystore lets you store `elasticsearch.yml` values by +name, the {beatname_uc} keystore lets you specify arbitrary names that you can +reference in the {beatname_uc} configuration. To create and manage keys, use the `keystore` command. See the <> for the full command syntax, including