WIP Feature/elasticsearch 6

build/scripts/Commandline.fsx

@@ -19,11 +19,10 @@ Targets:
 * clean
   - cleans build output folders
 * test [testfilter]
-  - incremental build and unit test for .NET 4.5, [testfilter] allows you to do
-    a contains match on the tests to be run.
+  - incremental build and unit test for .NET 4.5, [testfilter] allows you to do a contains match on the tests to be run.
 * release <version>
   - 0 create a release worthy nuget packages for [version] under build\output
-* integrate <elasticsearch_versions> [clustername] [testfilter]  -
+* integrate <elasticsearch_versions> [clustername] [testfilter] 
   - run integration tests for <elasticsearch_versions> which is a semicolon separated list of
     elasticsearch versions to test or `latest`. Can filter tests by <clustername> and <testfilter>
 * canary [apikey] [feed]
@@ -57,6 +56,14 @@ module Commandline =
         | ("test", _)
         | ("integrate", _) -> false
         | _ -> true
+
+    let needsClean =
+        match (target, skipTests) with
+        | (_, true) -> true
+        //dotnet-xunit needs to a build of its own anyways
+        | ("test", _)
+        | ("integrate", _) -> false
+        | _ -> true
 
     let arguments =
         match filteredArgs with

build/scripts/Targets.fsx

@@ -32,6 +32,7 @@ open Commandline
 Commandline.parse()
 
 Target "Build" <| fun _ -> traceHeader "STARTING BUILD"
+Target "Start" <| fun _ -> traceHeader "STARTING BUILD"
 
 Target "Clean" Build.Clean
 
@@ -75,7 +76,8 @@ Target "Canary" <| fun _ ->
     if (not (String.IsNullOrWhiteSpace apiKey) || apiKey = "ignore") then Release.PublishCanaryBuild apiKey feed
 
 // Dependencies
-"Clean" 
+"Start"
+  =?> ("Clean", Commandline.needsClean )
   =?> ("Version", hasBuildParam "version")
   ==> "Restore"
   =?> ("FullBuild", Commandline.needsFullBuild)
@@ -84,19 +86,22 @@ Target "Canary" <| fun _ ->
   ==> "Documentation"
   ==> "Build"
 
-"Clean"
+"Start"
+  =?> ("Clean", Commandline.needsClean )
   =?> ("FullBuild", Commandline.needsFullBuild)
   ==> "Profile"
 
-"Clean" 
+"Start"
+  =?> ("Clean", Commandline.needsClean )
   =?> ("FullBuild", Commandline.needsFullBuild)
   ==> "Benchmark"
 
 "Version"
   ==> "Release"
   ==> "Canary"
 
-"Clean"
+"Start"
+  =?> ("Clean", Commandline.needsClean )
   ==> "Restore"
   =?> ("FullBuild", Commandline.needsFullBuild)
   ==> "Integrate"

src/CodeGeneration/ApiGenerator/ApiGenerator.cs

@@ -41,6 +41,13 @@ public static void Generate(string downloadBranch, params string[] folders)
 			}
 		}
 
+		private static string[] IgnoredApis { get; } =
+		{
+			"xpack.ml.delete_filter.json",
+			"xpack.ml.get_filters.json",
+			"xpack.ml.put_filter.json",
+		};
+
 		private static RestApiSpec CreateRestApiSpecModel(string downloadBranch, string[] folders)
 		{
 			var directories = Directory.GetDirectories(CodeConfiguration.RestSpecificationFolder, "*", SearchOption.AllDirectories)
@@ -50,7 +57,12 @@ private static RestApiSpec CreateRestApiSpecModel(string downloadBranch, string[
 			var endpoints = new Dictionary<string, ApiEndpoint>();
 			using (var pbar = new ProgressBar(directories.Count, $"Listing {directories.Count} directories", new ProgressBarOptions { BackgroundColor = ConsoleColor.DarkGray }))
 			{
-				foreach (var jsonFiles in directories.Select(dir => Directory.GetFiles(dir).Where(f => f.EndsWith(".json")).ToList()))
+				var folderFiles = directories.Select(dir =>
+					Directory.GetFiles(dir)
+					.Where(f => f.EndsWith(".json") && !IgnoredApis.Contains(new FileInfo(f).Name))
+					.ToList()
+				);
+				foreach (var jsonFiles in folderFiles)
 				{
 					using (var fileProgress = pbar.Spawn(jsonFiles.Count, $"Listing {jsonFiles.Count} files", new ProgressBarOptions { ProgressCharacter = '─', BackgroundColor = ConsoleColor.DarkGray }))
 					{

src/CodeGeneration/ApiGenerator/Domain/ApiUrlPart.cs

@@ -46,6 +46,7 @@ public string ClrTypeName
 					case "filter_id":
 					case "id": return this.Type == "string" ? "Id" : "Ids";
 					case "category_id": return "CategoryId";
+					case "nodes":
 					case "node_id": return this.Type == "string" ? "NodeId" : "NodeIds";
 					case "scroll_id": return this.Type == "string" ? "ScrollId" : "ScrollIds";
 					case "field":
@@ -64,9 +65,11 @@ public string ClrTypeName
 					case "realm":
 					case "realms":
 					case "alias":
+					case "context":
 					case "name":
 					case "thread_pool_patterns":
 						return this.Type == "string" ? "Name" : "Names";
+					case "parent_task_id":
 					case "task_id": return "TaskId";
 					case "timestamp": return "Timestamp";
 					default: return this.Type + "_";

src/CodeGeneration/ApiGenerator/RestSpecification/Core/cat.segments.json

@@ -16,6 +16,11 @@
           "type" : "string",
           "description" : "a short version of the Accept header, e.g. json, yaml"
         },
+        "bytes": {
+          "type": "enum",
+          "description" : "The unit in which to display byte values",
+          "options": [ "b", "k", "kb", "m", "mb", "g", "gb", "t", "tb", "p", "pb" ]
+        },
         "h": {
             "type": "list",
             "description" : "Comma-separated list of column names to display"

src/CodeGeneration/ApiGenerator/RestSpecification/Core/cat.shards.json

@@ -16,6 +16,11 @@
           "type" : "string",
           "description" : "a short version of the Accept header, e.g. json, yaml"
         },
+        "bytes": {
+          "type": "enum",
+          "description" : "The unit in which to display byte values",
+          "options": [ "b", "k", "kb", "m", "mb", "g", "gb", "t", "tb", "p", "pb" ]
+        },
         "local": {
           "type" : "boolean",
           "description" : "Return local information, do not retrieve the state from master node (default: false)"

src/CodeGeneration/ApiGenerator/RestSpecification/Core/cat.snapshots.json

@@ -10,7 +10,6 @@
       "parts": {
         "repository": {
           "type" : "list",
-          "required": true,
           "description": "Name of repository from which to fetch the snapshot information"
         }
       },

src/CodeGeneration/ApiGenerator/RestSpecification/Core/remote.info.json → src/CodeGeneration/ApiGenerator/RestSpecification/Core/cluster.remote_info.json

@@ -1,6 +1,6 @@
 {
-  "remote.info": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/5.5/cluster-remote-info.html",
+  "cluster.remote_info": {
+    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-remote-info.html",
     "methods": ["GET"],
     "url": {
       "path": "/_remote/info",
@@ -9,4 +9,4 @@
     },
     "body": null
   }
-}
+}

src/CodeGeneration/ApiGenerator/RestSpecification/Core/count.json

@@ -39,8 +39,8 @@
           "description" : "Specify the node or shard the operation should be performed on (default: random)"
         },
         "routing": {
-          "type" : "string",
-          "description" : "Specific routing value"
+          "type" : "list",
+          "description" : "A comma-separated list of specific routing values"
         },
         "q": {
           "type" : "string",
@@ -67,6 +67,10 @@
         "lenient": {
           "type" : "boolean",
           "description" : "Specify whether format-based query failures (such as providing text to a numeric field) should be ignored"
+        },
+        "terminate_after" : {
+          "type" : "number",
+          "description" : "The maximum count for each shard, upon reaching which the query execution will terminate early"
         }
       }
     },

src/CodeGeneration/ApiGenerator/RestSpecification/Core/delete_script.json

@@ -3,21 +3,24 @@
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting.html",
     "methods": ["DELETE"],
     "url": {
-      "path": "/_scripts/{lang}",
-      "paths": [ "/_scripts/{lang}", "/_scripts/{lang}/{id}" ],
+      "path": "/_scripts/{id}",
+      "paths": [ "/_scripts/{id}" ],
       "parts": {
         "id": {
           "type" : "string",
           "description" : "Script ID",
           "required" : true
-        },
-        "lang" : {
-          "type" : "string",
-          "description" : "Script language",
-          "required" : true
         }
       },
       "params" : {
+        "timeout": {
+          "type" : "time",
+          "description" : "Explicit operation timeout"
+        },
+        "master_timeout": {
+          "type" : "time",
+          "description" : "Specify timeout for connection to master"
+        }
       }
     },
     "body": null

src/CodeGeneration/ApiGenerator/RestSpecification/Core/exists_source.json

@@ -1,6 +1,6 @@
 {
   "exists_source": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/5.5/docs-get.html",
+    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html",
     "methods": ["HEAD"],
     "url": {
       "path": "/{index}/{type}/{id}/_source",

src/CodeGeneration/ApiGenerator/RestSpecification/Core/field_caps.json

@@ -1,6 +1,6 @@
 {
   "field_caps": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/5.5/search-field-caps.html",
+    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-field-caps.html",
     "methods": ["GET", "POST"],
     "url": {
       "path": "/_field_caps",

src/CodeGeneration/ApiGenerator/RestSpecification/Core/get_script.json

@@ -3,18 +3,13 @@
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting.html",
     "methods": ["GET"],
     "url": {
-      "path": "/_scripts/{lang}",
-      "paths": [ "/_scripts/{lang}", "/_scripts/{lang}/{id}" ],
+      "path": "/_scripts/{id}",
+      "paths": [ "/_scripts/{id}" ],
       "parts": {
         "id": {
           "type" : "string",
           "description" : "Script ID",
           "required" : true
-        },
-        "lang" : {
-          "type" : "string",
-          "description" : "Script language",
-          "required" : true
         }
       },
       "params" : {

src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.delete.json

@@ -20,6 +20,20 @@
         "master_timeout": {
           "type" : "time",
           "description" : "Specify timeout for connection to master"
+        },
+        "ignore_unavailable": {
+          "type": "boolean",
+          "description": "Ignore unavailable indexes (default: false)"
+        },
+        "allow_no_indices": {
+          "type": "boolean",
+          "description": "Ignore if a wildcard expression resolves to no concrete indices (default: false)"
+        },
+        "expand_wildcards": {
+          "type": "enum",
+          "options": [ "open", "closed", "none", "all" ],
+          "default": "open",
+          "description": "Whether wildcard expressions should get expanded to open or closed indices (default: open)"
         }
       }
     },

src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.exists_template.json

@@ -8,7 +8,7 @@
       "parts": {
         "name": {
           "type": "list",
-          "required": false,
+          "required": true,
           "description": "The comma separated names of the index templates"
         }
       },

src/CodeGeneration/ApiGenerator/RestSpecification/Core/indices.get.json

@@ -4,17 +4,12 @@
     "methods":[ "GET" ],
     "url":{
       "path":"/{index}",
-      "paths":[ "/{index}", "/{index}/{feature}" ],
+      "paths":[ "/{index}" ],
       "parts":{
         "index":{
           "type":"list",
           "required" : true,
           "description":"A comma-separated list of index names"
-        },
-        "feature":{
-          "type":"list",
-          "description":"A comma-separated list of features",
-          "options": ["_settings", "_mappings", "_aliases"]
         }
       },
       "params":{

src/CodeGeneration/ApiGenerator/RestSpecification/Core/ingest.processor_grok.json

@@ -0,0 +1,15 @@
+{
+  "ingest.processor_grok": {
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/master/ingest.html",
+    "methods": [ "GET" ],
+    "url": {
+      "path": "/_ingest/processor/grok",
+      "paths": ["/_ingest/processor/grok"],
+      "parts": {
+      },
+      "params": {
+      }
+    },
+    "body": null
+  }
+}

src/CodeGeneration/ApiGenerator/RestSpecification/Core/msearch.json

@@ -28,6 +28,11 @@
         "typed_keys": {
           "type" : "boolean",
           "description" : "Specify whether aggregation and suggester names should be prefixed by their respective types in the response"
+        },
+        "pre_filter_shard_size" : {
+          "type" : "number",
+          "description" : "A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint.",
+          "default" : 128
         }
       }
     },