[Ingest] API completeness

[jfsiii]

Make sure all operations can be triggered through the API

[elasticmachine]

Pinging @elastic/ingest-management (Team:Ingest Management)

[nchaulet]

I took a look at this, and all the actions we are doing through the UI are doable with API.

I still have a concern for one action, adding a new datasource to an agent config, the API call you need to made are not obvious and need a few transformations client side:

For example to add nginx you need to get the package:

GET http://localhost:5601/api/ingest_manager/epm/packages/nginx-0.0.2

See the response

{
"response": {
  "format_version": "1.0.0",
  "name": "nginx",
  "title": "Nginx",
  "version": "0.0.2",
  "readme": "/package/nginx/0.0.2/docs/README.md",
  "license": "basic",
  "description": "Nginx integration",
  "type": "integration",
  "categories": [
    "logs",
    "metrics"
  ],
  "release": "ga",
  "removable": true,
  "requirement": {
    "kibana": {
      "versions": ">7.1.0 <7.6.0"
    },
    "elasticsearch": {
      "versions": ">7.0.1"
    }
  },
  "screenshots": [
    {
      "src": "/package/nginx/0.0.2/img/metrics.png",
      "title": "Nginx metrics overview",
      "size": "2560x2100"
    },
    {
      "src": "/package/nginx/0.0.2/img/logs.png",
      "title": "Nginx logs overview",
      "size": "1218x1266"
    }
  ],
  "icons": [
    {
      "src": "/package/nginx/0.0.2/img/icon.svg",
      "type": "image/svg+xml"
    }
  ],
  "assets": {
    "kibana": {
      "dashboard": [
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "dashboard",
          "file": "046212a0-a2a1-11e7-928f-5dbe6f6f5519-ecs.json",
          "path": "nginx-0.0.2/kibana/dashboard/046212a0-a2a1-11e7-928f-5dbe6f6f5519-ecs.json"
        },
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "dashboard",
          "file": "55a9e6e0-a29e-11e7-928f-5dbe6f6f5519-ecs.json",
          "path": "nginx-0.0.2/kibana/dashboard/55a9e6e0-a29e-11e7-928f-5dbe6f6f5519-ecs.json"
        },
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "dashboard",
          "file": "023d2930-f1a5-11e7-a9ef-93c69af7b129-ecs.json",
          "path": "nginx-0.0.2/kibana/dashboard/023d2930-f1a5-11e7-a9ef-93c69af7b129-ecs.json"
        }
      ],
      "search": [
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "search",
          "file": "Filebeat-Nginx-module-ecs.json",
          "path": "nginx-0.0.2/kibana/search/Filebeat-Nginx-module-ecs.json"
        },
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "search",
          "file": "9eb25600-a1f0-11e7-928f-5dbe6f6f5519-ecs.json",
          "path": "nginx-0.0.2/kibana/search/9eb25600-a1f0-11e7-928f-5dbe6f6f5519-ecs.json"
        },
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "search",
          "file": "6d9e66d0-a1f0-11e7-928f-5dbe6f6f5519-ecs.json",
          "path": "nginx-0.0.2/kibana/search/6d9e66d0-a1f0-11e7-928f-5dbe6f6f5519-ecs.json"
        }
      ],
      "visualization": [
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "visualization",
          "file": "dcbffe30-f1a4-11e7-a9ef-93c69af7b129-ecs.json",
          "path": "nginx-0.0.2/kibana/visualization/dcbffe30-f1a4-11e7-a9ef-93c69af7b129-ecs.json"
        },
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "visualization",
          "file": "0dd6f320-a29f-11e7-928f-5dbe6f6f5519-ecs.json",
          "path": "nginx-0.0.2/kibana/visualization/0dd6f320-a29f-11e7-928f-5dbe6f6f5519-ecs.json"
        },
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "visualization",
          "file": "a1d92240-f1a1-11e7-a9ef-93c69af7b129-ecs.json",
          "path": "nginx-0.0.2/kibana/visualization/a1d92240-f1a1-11e7-a9ef-93c69af7b129-ecs.json"
        },
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "visualization",
          "file": "Nginx-Access-OSes-ecs.json",
          "path": "nginx-0.0.2/kibana/visualization/Nginx-Access-OSes-ecs.json"
        },
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "visualization",
          "file": "1cfb1a80-a1f4-11e7-928f-5dbe6f6f5519-ecs.json",
          "path": "nginx-0.0.2/kibana/visualization/1cfb1a80-a1f4-11e7-928f-5dbe6f6f5519-ecs.json"
        },
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "visualization",
          "file": "555df8a0-f1a1-11e7-a9ef-93c69af7b129-ecs.json",
          "path": "nginx-0.0.2/kibana/visualization/555df8a0-f1a1-11e7-a9ef-93c69af7b129-ecs.json"
        },
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "visualization",
          "file": "Nginx-Access-Map-ecs.json",
          "path": "nginx-0.0.2/kibana/visualization/Nginx-Access-Map-ecs.json"
        },
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "visualization",
          "file": "b70b1b20-a1f4-11e7-928f-5dbe6f6f5519-ecs.json",
          "path": "nginx-0.0.2/kibana/visualization/b70b1b20-a1f4-11e7-928f-5dbe6f6f5519-ecs.json"
        },
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "visualization",
          "file": "46322e50-a1f6-11e7-928f-5dbe6f6f5519-ecs.json",
          "path": "nginx-0.0.2/kibana/visualization/46322e50-a1f6-11e7-928f-5dbe6f6f5519-ecs.json"
        },
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "visualization",
          "file": "d763a570-f1a1-11e7-a9ef-93c69af7b129-ecs.json",
          "path": "nginx-0.0.2/kibana/visualization/d763a570-f1a1-11e7-a9ef-93c69af7b129-ecs.json"
        },
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "visualization",
          "file": "Nginx-Access-Browsers-ecs.json",
          "path": "nginx-0.0.2/kibana/visualization/Nginx-Access-Browsers-ecs.json"
        },
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "visualization",
          "file": "9184fa00-a1f5-11e7-928f-5dbe6f6f5519-ecs.json",
          "path": "nginx-0.0.2/kibana/visualization/9184fa00-a1f5-11e7-928f-5dbe6f6f5519-ecs.json"
        },
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "visualization",
          "file": "97109780-a2a5-11e7-928f-5dbe6f6f5519-ecs.json",
          "path": "nginx-0.0.2/kibana/visualization/97109780-a2a5-11e7-928f-5dbe6f6f5519-ecs.json"
        },
        {
          "pkgkey": "nginx-0.0.2",
          "service": "kibana",
          "type": "visualization",
          "file": "47a8e0f0-f1a4-11e7-a9ef-93c69af7b129-ecs.json",
          "path": "nginx-0.0.2/kibana/visualization/47a8e0f0-f1a4-11e7-a9ef-93c69af7b129-ecs.json"
        }
      ]
    }
  },
  "datasets": [
    {
      "id": "nginx.access",
      "title": "Nginx Acess Logs",
      "release": "beta",
      "type": "logs",
      "ingest_pipeline": "default",
      "streams": [
        {
          "input": "logs",
          "vars": [
            {
              "name": "paths",
              "type": "text",
              "title": "Access log paths",
              "description": "Paths to the nginx access log file.",
              "multi": true,
              "required": true,
              "show_user": true,
              "default": [
                "/var/log/nginx/access.log*"
              ],
              "os": {
                "darwin": {
                  "default": [
                    "/usr/local/var/log/nginx/access.log*"
                  ]
                },
                "windows": {
                  "default": [
                    "c:/programdata/nginx/logs/*access.log*"
                  ]
                }
              }
            }
          ],
          "title": "Nginx access logs",
          "description": "Collecting the nginx access logs from file."
        }
      ],
      "package": "nginx",
      "path": "access"
    },
    {
      "id": "nginx.error",
      "title": "Nginx Error Logs",
      "release": "experimental",
      "type": "logs",
      "ingest_pipeline": "pipeline",
      "streams": [
        {
          "input": "logs",
          "vars": [
            {
              "name": "paths",
              "type": "",
              "multi": false,
              "required": true,
              "show_user": false,
              "default": [
                "/var/log/nginx/error.log*"
              ],
              "os": {
                "darwin": [
                  "/usr/local/var/log/nginx/error.log*"
                ],
                "windows": [
                  "c:/programdata/nginx/logs/error.log*"
                ]
              }
            }
          ],
          "template_path": "stream.yml.hbs",
          "title": "Nginx error logs",
          "description": "Collecting the Nginx error logs from file."
        },
        {
          "input": "syslog",
          "vars": [
            {
              "name": "protocol.udp.host",
              "type": "",
              "multi": false,
              "required": true,
              "show_user": false,
              "default": [
                "localhost:9000"
              ]
            }
          ],
          "template_path": "syslog.yml.hbs",
          "title": "Nginx error logs",
          "description": "Collecting the Nginx error logs from syslog."
        }
      ],
      "package": "nginx",
      "path": "error"
    },
    {
      "id": "nginx.stubstatus",
      "title": "Nginx stubstatus metrics",
      "release": "beta",
      "type": "metrics",
      "streams": [
        {
          "input": "nginx/metrics",
          "vars": [
            {
              "name": "period",
              "type": "duration",
              "description": "Collection period. Valid values: 10s, 5m, 2h",
              "multi": false,
              "required": false,
              "show_user": false,
              "default": "10s"
            }
          ],
          "title": "Nginx stubstatus metrics",
          "description": "Collecting [stubstatus metrics](http://nginx.org/en/docs/http/ngx_http_stub_status_module.html) from the nginx service."
        }
      ],
      "package": "nginx",
      "path": "stubstatus"
    }
  ],
  "datasources": [
    {
      "name": "nginx",
      "title": "Nginx logs and metrics.",
      "description": "Collecting logs and metrics from nginx.",
      "inputs": [
        {
          "type": "nginx/metrics",
          "vars": [
            {
              "name": "hosts",
              "type": "text",
              "title": "Hosts",
              "description": "List of nginx hosts to collect data from.",
              "multi": true,
              "required": true,
              "show_user": false,
              "default": [
                "http://127.0.0.1"
              ]
            },
            {
              "name": "username",
              "type": "text",
              "title": "Username",
              "description": "Username to authenticate to the Nginx hosts",
              "multi": false,
              "required": false,
              "show_user": false
            },
            {
              "name": "password",
              "type": "password",
              "title": "Password",
              "description": "Password to authenticate to the Nginx hosts",
              "multi": false,
              "required": false,
              "show_user": false
            }
          ],
          "title": "Collect metrics from nginx hosts",
          "description": "Collect stubstatus metrics from nginx hosts.",
          "streams": [
            {
              "input": "nginx/metrics",
              "vars": [
                {
                  "name": "period",
                  "type": "duration",
                  "description": "Collection period. Valid values: 10s, 5m, 2h",
                  "multi": false,
                  "required": false,
                  "show_user": false,
                  "default": "10s"
                }
              ],
              "dataset": "nginx.stubstatus",
              "template_path": "stream.yml.hbs",
              "template": "# Defines which input to use, should be stripped out when creating the config\ninput: nginx/metrics\nmetricsets: [\"stubstatus\"]\nperiod: {{period}}\n\n# Nginx hosts\nhosts:\n{{#each hosts}}\n  - {{this}}\n{{/each}}\n\n# Path to server status. Default server-status\n#server_status_path: \"server-status\"\n\n{{#if username}}\nusername: \"{{username}}\"\n{{/if}}\n{{#if password}}\npassword: \"{{password}}\"\n{{/if}}\n",
              "title": "Nginx stubstatus metrics",
              "description": "Collecting [stubstatus metrics](http://nginx.org/en/docs/http/ngx_http_stub_status_module.html) from the nginx service."
            }
          ]
        },
        {
          "type": "logs",
          "title": "Collect nginx logs",
          "description": "Collect access and error logs from nginx.",
          "streams": [
            {
              "input": "logs",
              "vars": [
                {
                  "name": "paths",
                  "type": "text",
                  "title": "Access log paths",
                  "description": "Paths to the nginx access log file.",
                  "multi": true,
                  "required": true,
                  "show_user": true,
                  "default": [
                    "/var/log/nginx/access.log*"
                  ],
                  "os": {
                    "darwin": {
                      "default": [
                        "/usr/local/var/log/nginx/access.log*"
                      ]
                    },
                    "windows": {
                      "default": [
                        "c:/programdata/nginx/logs/*access.log*"
                      ]
                    }
                  }
                }
              ],
              "dataset": "nginx.access",
              "template_path": "stream.yml.hbs",
              "template": "\ninput: log\npaths:\n{{#each paths}}\n  - \"{{this}}\"\n{{/each}}\n\nexclude_files: [\".gz$\"]\n\nprocessors:\n  - add_locale: ~\n",
              "title": "Nginx access logs",
              "description": "Collecting the nginx access logs from file."
            },
            {
              "input": "logs",
              "vars": [
                {
                  "name": "paths",
                  "type": "",
                  "multi": false,
                  "required": true,
                  "show_user": false,
                  "default": [
                    "/var/log/nginx/error.log*"
                  ],
                  "os": {
                    "darwin": [
                      "/usr/local/var/log/nginx/error.log*"
                    ],
                    "windows": [
                      "c:/programdata/nginx/logs/error.log*"
                    ]
                  }
                }
              ],
              "dataset": "nginx.error",
              "template_path": "stream.yml.hbs",
              "template": "input: log\n\n{{#each paths}}\npaths: \"{{this}}\"\n{{/each}}\nexclude_files: [\".gz$\"]\n\nprocessors:\n  - add_locale: ~\n",
              "title": "Nginx error logs",
              "description": "Collecting the Nginx error logs from file."
            }
          ]
        }
      ],
      "multiple": true
    }
  ],
  "download": "/epr/nginx/nginx-0.0.2.tar.gz",
  "path": "/package/nginx/0.0.2",
  "latestVersion": "0.0.2",
  "status": "not_installed"
},
"success": true
}

And from that create the datasource

POST http://localhost:5601/api/ingest_manager/datasources

See the request

{
  "name": "nginx-1",
  "description": "",
  "config_id": "aa321a90-8ed5-11ea-88fe-6ba145d0b0de",
  "enabled": true,
  "output_id": "",
  "inputs": [
    {
      "type": "nginx/metrics",
      "enabled": true,
      "streams": [
        {
          "id": "nginx/metrics-nginx.stubstatus",
          "enabled": true,
          "dataset": "nginx.stubstatus",
          "vars": {
            "period": {
              "value": "10s",
              "type": "duration"
            }
          }
        }
      ],
      "vars": {
        "hosts": {
          "value": [
            "http://127.0.0.1"
          ],
          "type": "text"
        },
        "username": {
          "type": "text"
        },
        "password": {
          "type": "password"
        }
      }
    },
    {
      "type": "logs",
      "enabled": true,
      "streams": [
        {
          "id": "logs-nginx.access",
          "enabled": true,
          "dataset": "nginx.access",
          "vars": {
            "paths": {
              "value": [
                "/var/log/nginx/access.log*"
              ],
              "type": "text"
            }
          }
        },
        {
          "id": "logs-nginx.error",
          "enabled": true,
          "dataset": "nginx.error",
          "vars": {
            "paths": {
              "value": [
                "/var/log/nginx/error.log*"
              ]
            }
          }
        }
      ]
    }
  ],
  "namespace": "default",
  "package": {
    "name": "nginx",
    "title": "Nginx",
    "version": "0.0.2"
  }
}

I am wondering if in the future we can simplify that?

[jfsiii]

#68323 can help with this if we want. For example it can be used to generate mock servers which can be used in testing the API

It's also possible to use these in e2e tests like elastic/e2e-testing#132 in some way (looking for a path by name get-fleet-setup and getting info about what it accepts, returns, etc)

[ph]

Are we good or we are missing action that cannot be executed through the api?

[jen-huang]

Closing due to staleness, we haven't had anything come up in way of lacking API functionality (apart from documenting them 😉).