Apply release fallback to datastreams validation (#804)

elastic · Apr 4, 2022 · d45d98f · d45d98f
1 parent 158b20f
commit d45d98f
Show file tree

Hide file tree

Showing 21 changed files with 908 additions and 5 deletions.
diff --git a/packages/datastream.go b/packages/datastream.go
@@ -174,7 +174,7 @@ func NewDataStream(basePath string, p *Package) (*DataStream, error) {
 	}
 
 	if !IsValidRelease(d.Release) {
-		return nil, fmt.Errorf("invalid release: %s", d.Release)
+		return nil, fmt.Errorf("invalid release: %q", d.Release)
 	}
 
 	pipelineDir := filepath.Join(d.BasePath, "elasticsearch", DirIngestPipeline)

diff --git a/packages/package.go b/packages/package.go
@@ -282,7 +282,7 @@ func NewPackage(basePath string, fsBuilder FileSystemBuilder) (*Package, error)
 	}
 
 	if !IsValidRelease(p.Release) {
-		return nil, fmt.Errorf("invalid release: %s", p.Release)
+		return nil, fmt.Errorf("invalid release: %q", p.Release)
 	}
 
 	readmePath := filepath.Join("docs", "README.md")
@@ -441,11 +441,15 @@ func (p *Package) Validate() error {
 		return fmt.Errorf("invalid package version: %s, %s", p.FormatVersion, err)
 	}
 
-	_, err = semver.StrictNewVersion(p.Version)
+	p.versionSemVer, err = semver.StrictNewVersion(p.Version)
 	if err != nil {
 		return err
 	}
 
+	if p.Release == "" {
+		p.Release = releaseForSemVerCompat(p.versionSemVer)
+	}
+
 	if p.Title == nil || *p.Title == "" {
 		return fmt.Errorf("no title set for package: %s", p.Name)
 	}

diff --git a/testdata/generated/categories-experimental.json b/testdata/generated/categories-experimental.json
@@ -29,6 +29,11 @@
     "title": "Custom",
     "count": 14
   },
+  {
+    "id": "datastore",
+    "title": "Datastore",
+    "count": 1
+  },
   {
     "id": "message_queue",
     "title": "Message Queue",
@@ -37,7 +42,7 @@
   {
     "id": "monitoring",
     "title": "Monitoring",
-    "count": 2
+    "count": 3
   },
   {
     "id": "web",

diff --git a/testdata/generated/categories-prerelease.json b/testdata/generated/categories-prerelease.json
@@ -29,6 +29,11 @@
     "title": "Custom",
     "count": 14
   },
+  {
+    "id": "datastore",
+    "title": "Datastore",
+    "count": 1
+  },
   {
     "id": "message_queue",
     "title": "Message Queue",
@@ -37,7 +42,7 @@
   {
     "id": "monitoring",
     "title": "Monitoring",
-    "count": 1
+    "count": 2
   },
   {
     "id": "web",

diff --git a/testdata/generated/package/datastream_without_release/0.1.0/index.json b/testdata/generated/package/datastream_without_release/0.1.0/index.json
@@ -0,0 +1,121 @@
+{
+  "name": "datastream_without_release",
+  "title": "Apache Spark",
+  "version": "0.1.0",
+  "release": "beta",
+  "description": "Collect metrics from Apache Spark with Elastic Agent.",
+  "type": "integration",
+  "download": "/epr/datastream_without_release/datastream_without_release-0.1.0.zip",
+  "path": "/package/datastream_without_release/0.1.0",
+  "icons": [
+    {
+      "src": "/img/apache_spark-logo.svg",
+      "path": "/package/datastream_without_release/0.1.0/img/apache_spark-logo.svg",
+      "title": "Apache Spark logo",
+      "size": "32x32",
+      "type": "image/svg+xml"
+    }
+  ],
+  "conditions": {
+    "kibana": {
+      "version": "^8.1.0"
+    }
+  },
+  "owner": {
+    "github": "elastic/obs-service-integrations"
+  },
+  "categories": [
+    "datastore",
+    "monitoring"
+  ],
+  "format_version": "1.0.0",
+  "readme": "/package/datastream_without_release/0.1.0/docs/README.md",
+  "license": "basic",
+  "screenshots": [
+    {
+      "src": "/img/apache_spark-screenshot.png",
+      "path": "/package/datastream_without_release/0.1.0/img/apache_spark-screenshot.png",
+      "title": "Apache Spark screenshot",
+      "size": "600x600",
+      "type": "image/png"
+    }
+  ],
+  "assets": [
+    "/package/datastream_without_release/0.1.0/changelog.yml",
+    "/package/datastream_without_release/0.1.0/manifest.yml",
+    "/package/datastream_without_release/0.1.0/docs/README.md",
+    "/package/datastream_without_release/0.1.0/img/apache_spark-logo.svg",
+    "/package/datastream_without_release/0.1.0/img/apache_spark-screenshot.png",
+    "/package/datastream_without_release/0.1.0/data_stream/nodes/manifest.yml",
+    "/package/datastream_without_release/0.1.0/data_stream/nodes/sample_event.json",
+    "/package/datastream_without_release/0.1.0/data_stream/nodes/fields/base-fields.yml",
+    "/package/datastream_without_release/0.1.0/data_stream/nodes/fields/ecs.yml",
+    "/package/datastream_without_release/0.1.0/data_stream/nodes/fields/fields.yml",
+    "/package/datastream_without_release/0.1.0/data_stream/nodes/agent/stream/stream.yml.hbs",
+    "/package/datastream_without_release/0.1.0/data_stream/nodes/elasticsearch/ingest_pipeline/default.yml"
+  ],
+  "policy_templates": [
+    {
+      "name": "apache_spark",
+      "title": "Apache Spark metrics",
+      "description": "Collect Apache Spark metrics",
+      "inputs": [
+        {
+          "type": "jolokia/metrics",
+          "title": "Collect Apache Spark Metrics using Jolokia",
+          "description": "Collecting metrics from Apache Spark."
+        }
+      ],
+      "multiple": true
+    }
+  ],
+  "data_streams": [
+    {
+      "type": "metrics",
+      "dataset": "datastream_without_release.nodes",
+      "title": "Apache Spark nodes metrics",
+      "release": "beta",
+      "ingest_pipeline": "default",
+      "streams": [
+        {
+          "input": "jolokia/metrics",
+          "vars": [
+            {
+              "name": "hosts",
+              "type": "text",
+              "title": "Hosts",
+              "description": "Full hosts for the Jolokia for Apache Spark (https://spark_main:jolokia_port).\n",
+              "multi": true,
+              "required": true,
+              "show_user": true
+            },
+            {
+              "name": "path",
+              "type": "text",
+              "title": "Path",
+              "multi": false,
+              "required": true,
+              "show_user": false,
+              "default": "/jolokia/?ignoreErrors=true&canonicalNaming=false"
+            },
+            {
+              "name": "period",
+              "type": "text",
+              "title": "Period",
+              "multi": false,
+              "required": true,
+              "show_user": true,
+              "default": "60s"
+            }
+          ],
+          "template_path": "stream.yml.hbs",
+          "title": "Apache Spark nodes metrics",
+          "description": "Collect Apache Spark nodes metrics using Jolokia agent.",
+          "enabled": true
+        }
+      ],
+      "package": "datastream_without_release",
+      "path": "nodes"
+    }
+  ]
+}
diff --git a/testdata/generated/search-category-datastore-prerelease.json b/testdata/generated/search-category-datastore-prerelease.json
@@ -1,4 +1,42 @@
 [
+  {
+    "name": "datastream_without_release",
+    "title": "Apache Spark",
+    "version": "0.1.0",
+    "release": "beta",
+    "description": "Collect metrics from Apache Spark with Elastic Agent.",
+    "type": "integration",
+    "download": "/epr/datastream_without_release/datastream_without_release-0.1.0.zip",
+    "path": "/package/datastream_without_release/0.1.0",
+    "icons": [
+      {
+        "src": "/img/apache_spark-logo.svg",
+        "path": "/package/datastream_without_release/0.1.0/img/apache_spark-logo.svg",
+        "title": "Apache Spark logo",
+        "size": "32x32",
+        "type": "image/svg+xml"
+      }
+    ],
+    "policy_templates": [
+      {
+        "name": "apache_spark",
+        "title": "Apache Spark metrics",
+        "description": "Collect Apache Spark metrics"
+      }
+    ],
+    "conditions": {
+      "kibana": {
+        "version": "^8.1.0"
+      }
+    },
+    "owner": {
+      "github": "elastic/obs-service-integrations"
+    },
+    "categories": [
+      "datastore",
+      "monitoring"
+    ]
+  },
   {
     "name": "example",
     "title": "Example Integration",

diff --git a/testdata/generated/search-category-datastore.json b/testdata/generated/search-category-datastore.json
@@ -1,4 +1,42 @@
 [
+  {
+    "name": "datastream_without_release",
+    "title": "Apache Spark",
+    "version": "0.1.0",
+    "release": "beta",
+    "description": "Collect metrics from Apache Spark with Elastic Agent.",
+    "type": "integration",
+    "download": "/epr/datastream_without_release/datastream_without_release-0.1.0.zip",
+    "path": "/package/datastream_without_release/0.1.0",
+    "icons": [
+      {
+        "src": "/img/apache_spark-logo.svg",
+        "path": "/package/datastream_without_release/0.1.0/img/apache_spark-logo.svg",
+        "title": "Apache Spark logo",
+        "size": "32x32",
+        "type": "image/svg+xml"
+      }
+    ],
+    "policy_templates": [
+      {
+        "name": "apache_spark",
+        "title": "Apache Spark metrics",
+        "description": "Collect Apache Spark metrics"
+      }
+    ],
+    "conditions": {
+      "kibana": {
+        "version": "^8.1.0"
+      }
+    },
+    "owner": {
+      "github": "elastic/obs-service-integrations"
+    },
+    "categories": [
+      "datastore",
+      "monitoring"
+    ]
+  },
   {
     "name": "example",
     "title": "Example Integration",

diff --git a/testdata/generated/search-package-experimental.json b/testdata/generated/search-package-experimental.json
@@ -1,4 +1,42 @@
 [
+  {
+    "name": "datastream_without_release",
+    "title": "Apache Spark",
+    "version": "0.1.0",
+    "release": "beta",
+    "description": "Collect metrics from Apache Spark with Elastic Agent.",
+    "type": "integration",
+    "download": "/epr/datastream_without_release/datastream_without_release-0.1.0.zip",
+    "path": "/package/datastream_without_release/0.1.0",
+    "icons": [
+      {
+        "src": "/img/apache_spark-logo.svg",
+        "path": "/package/datastream_without_release/0.1.0/img/apache_spark-logo.svg",
+        "title": "Apache Spark logo",
+        "size": "32x32",
+        "type": "image/svg+xml"
+      }
+    ],
+    "policy_templates": [
+      {
+        "name": "apache_spark",
+        "title": "Apache Spark metrics",
+        "description": "Collect Apache Spark metrics"
+      }
+    ],
+    "conditions": {
+      "kibana": {
+        "version": "^8.1.0"
+      }
+    },
+    "owner": {
+      "github": "elastic/obs-service-integrations"
+    },
+    "categories": [
+      "datastore",
+      "monitoring"
+    ]
+  },
   {
     "name": "dataset_is_prefix",
     "title": "DatasetIsPrefix Flag",

diff --git a/testdata/generated/search-package-prerelease.json b/testdata/generated/search-package-prerelease.json
@@ -1,4 +1,42 @@
 [
+  {
+    "name": "datastream_without_release",
+    "title": "Apache Spark",
+    "version": "0.1.0",
+    "release": "beta",
+    "description": "Collect metrics from Apache Spark with Elastic Agent.",
+    "type": "integration",
+    "download": "/epr/datastream_without_release/datastream_without_release-0.1.0.zip",
+    "path": "/package/datastream_without_release/0.1.0",
+    "icons": [
+      {
+        "src": "/img/apache_spark-logo.svg",
+        "path": "/package/datastream_without_release/0.1.0/img/apache_spark-logo.svg",
+        "title": "Apache Spark logo",
+        "size": "32x32",
+        "type": "image/svg+xml"
+      }
+    ],
+    "policy_templates": [
+      {
+        "name": "apache_spark",
+        "title": "Apache Spark metrics",
+        "description": "Collect Apache Spark metrics"
+      }
+    ],
+    "conditions": {
+      "kibana": {
+        "version": "^8.1.0"
+      }
+    },
+    "owner": {
+      "github": "elastic/obs-service-integrations"
+    },
+    "categories": [
+      "datastore",
+      "monitoring"
+    ]
+  },
   {
     "name": "dataset_is_prefix",
     "title": "DatasetIsPrefix Flag",

diff --git a/testdata/package/datastream_without_release/0.1.0/changelog.yml b/testdata/package/datastream_without_release/0.1.0/changelog.yml
@@ -0,0 +1,7 @@
+# newer versions go on top
+
+- version: "0.1.0"
+  changes:
+    - description: Implement "nodes" data stream
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2939