Add ML APIs to client

- Unzip ML integration test data tar archives using SharpZipLib. Add net45 compatible version to Tests
- Derive XPackMachineLearningCluster from XPackCluster that seeds ML data and includes additional methods for testing ML APIs
- Skip ML integration tests for versions < 5.4.0
- Introduce IntegrationTeardown method to close jobs that may be opened as part of tests
- Increase concurrent ML job allocations and max open jobs when starting ML cluster
- Explain why certain URL parts are not replaced during code generation
    Remove special treatment for task_id. This no longer seems to be needed.

- Specify node startup timeout on ClusterBase
    starting up an ML cluster with Fiddler attached could take some time so bump the start timeout for it.
- Increase the FAKE process tooling timeout

build/scripts/Tooling.fsx

@@ -69,7 +69,7 @@ module Tooling =
             ) timeout
         code
 
-    let private defaultTimeout = TimeSpan.FromMinutes 15.0
+    let private defaultTimeout = TimeSpan.FromMinutes 20.0
 
     let execProcessInDirectory proc arguments workingDir =
         let exitCode = execProcessWithTimeout proc arguments defaultTimeout workingDir
@@ -179,6 +179,7 @@ module Tooling =
                     Targets = ["Build"]
                     Properties =
                         [
+                            "OutputPathBaseDir", Path.GetFullPath "build\\output"
                             "Optimize", "True"
                             "Configuration", "Release"
                             "TargetFrameworkVersion", identifier.MSBuild

paket-files/paket.restore.cached

@@ -0,0 +1,14 @@
+
+
+GROUP build
+NUGET
+  remote: https://www.nuget.org/api/v2
+    Elasticsearch.Net (5.3) - framework: net45
+    FAKE (4.57.4)
+    FSharp.Data (2.3.2)
+      Zlib.Portable (>= 1.11) - framework: >= netstandard11, portable-net45+sl5+win8, portable-net45+win8, portable-net45+win8+wp8+wpa81
+    NEST (5.3) - framework: net45
+      Elasticsearch.Net (>= 5.3 < 6.0) - framework: net45, >= net46, >= netstandard13
+      Newtonsoft.Json (>= 9.0 < 10.0) - framework: net45, >= net46, >= netstandard13
+    Newtonsoft.Json (9.0.1) - framework: net45
+    Zlib.Portable (1.11) - framework: >= netstandard11, portable-net45+sl5+win8, portable-net45+win8, portable-net45+win8+wp8+wpa81

src/CodeGeneration/ApiGenerator/ApiGenerator.cs

@@ -59,6 +59,7 @@ private static RestApiSpec CreateRestApiSpecModel(string downloadBranch, string[
 							if (file.EndsWith("_common.json")) RestApiSpec.CommonApiQueryParameters = CreateCommonApiQueryParameters(file);
 							else if (file.EndsWith(".obsolete.json")) continue;
 							else if (file.EndsWith(".patch.json")) continue;
+							else if (file.EndsWith(".replace.json")) continue;
 							else
 							{
 								var endpoint = CreateApiEndpoint(file);
@@ -74,8 +75,6 @@ private static RestApiSpec CreateRestApiSpecModel(string downloadBranch, string[
 			return new RestApiSpec { Endpoints = endpoints, Commit = downloadBranch };
 		}
 
-
-
 		public static string PascalCase(string s)
 		{
 			var textInfo = new CultureInfo("en-US").TextInfo;
@@ -84,6 +83,15 @@ public static string PascalCase(string s)
 
 		private static KeyValuePair<string, ApiEndpoint> CreateApiEndpoint(string jsonFile)
 		{
+			var replaceFile = Path.Combine(Path.GetDirectoryName(jsonFile), Path.GetFileNameWithoutExtension(jsonFile)) + ".replace.json";
+			if (File.Exists(replaceFile))
+			{
+				var replaceSpec = JObject.Parse(File.ReadAllText(replaceFile));
+				var endpointReplaced = replaceSpec.ToObject<Dictionary<string, ApiEndpoint>>().First();
+				endpointReplaced.Value.CsharpMethodName = CreateMethodName(endpointReplaced.Key);
+				return endpointReplaced;
+			}
+
 			var officialJsonSpec = JObject.Parse(File.ReadAllText(jsonFile));
 			PatchOfficialSpec(officialJsonSpec, jsonFile);
 			var endpoint = officialJsonSpec.ToObject<Dictionary<string, ApiEndpoint>>().First();

src/CodeGeneration/ApiGenerator/Domain/ApiEndpoint.cs

@@ -17,7 +17,20 @@ public IEnumerable<string> MethodArguments
 			get
 			{
 				var methodArgs = CsharpMethod.Parts
-					.Select(p => p.Name != "body" ? "p.RouteValues." + p.Name.ToPascalCase() + (p.Type == "enum" ? ".Value" : "") : "body")
+					.Select(p =>
+					{
+						if (p.Name == "body") return "body";
+
+						switch (p.Type)
+						{
+							case "enum":
+								return $"p.RouteValues.{p.Name.ToPascalCase()}.Value";
+							case "long":
+								return $"long.Parse(p.RouteValues.{p.Name.ToPascalCase()})";
+							default:
+								return $"p.RouteValues.{p.Name.ToPascalCase()}";
+						}
+					})
 					.Concat(new[] {"u => p.RequestParameters"});
 				return methodArgs;
 			}

src/CodeGeneration/ApiGenerator/Domain/ApiQueryParameters.cs

@@ -40,6 +40,8 @@ public string CsharpType(string paramName)
 				case "":
 				case null:
 					return "string";
+				case "date":
+					return "DateTimeOffset";
 				case "enum":
 					return paramName.ToPascalCase();
 				default:

src/CodeGeneration/ApiGenerator/Domain/ApiUrlPart.cs

@@ -10,7 +10,11 @@ public class ApiUrlPart
 
 
 		private string _description;
-		public string Description { get { return _description; } set { _description = CleanUpDescription(value); } }
+		public string Description
+		{
+			get => _description;
+			set => _description = CleanUpDescription(value);
+		}
 		public bool Required { get; set; }
 		public IEnumerable<string> Options { get; set; }
 
@@ -36,7 +40,12 @@ public string ClrTypeName
 						return "IndexName";
 					case "type": return this.Type == "string" ? "TypeName" : "Types";
 					case "watch_id":
+					case "job_id":
+					case "datafeed_id":
+					case "snapshot_id":
+					case "filter_id":
 					case "id": return this.Type == "string" ? "Id" : "Ids";
+					case "category_id": return "CategoryId";
 					case "node_id": return this.Type == "string" ? "NodeId" : "NodeIds";
 					case "scroll_id": return this.Type == "string" ? "ScrollId" : "ScrollIds";
 					case "field":
@@ -59,6 +68,7 @@ public string ClrTypeName
 					case "thread_pool_patterns":
 						return this.Type == "string" ? "Name" : "Names";
 					case "task_id": return "TaskId";
+					case "timestamp": return "Timestamp";
 					default: return this.Type + "_";
 				}
 			}

src/CodeGeneration/ApiGenerator/Domain/CsharpMethod.cs

@@ -226,6 +226,13 @@ public IEnumerable<Constructor> DescriptorConstructors()
 					generated = $"public {m}({par}) : base(r => r.Required(\"index\", (Indices)typeof({generic}))){{}}";
 				}
 
+				// Use generic T to set the Indices and Types by default in the ctor
+				if (m == "PutDatafeedDescriptor" || m == "UpdateDatafeedDescriptor")
+				{
+					doc = AppendToSummary(doc, ". Will infer the index and type from the generic type");
+					generated = $"public {m}({par}) : base({routing}){{ Self.Indices = typeof({this.CallTypeGeneric}); Self.Types = typeof({this.CallTypeGeneric}); }}";
+				}
+
 				var c = new Constructor { Generated = generated, Description = doc };
 				ctors.Add(c);
 			}

src/CodeGeneration/ApiGenerator/Overrides/Descriptors/FlushJobDescriptorOverrides.cs

@@ -0,0 +1,15 @@
+using System.Collections.Generic;
+
+namespace ApiGenerator.Overrides.Descriptors
+{
+	public class FlushJobDescriptorOverrides : DescriptorOverridesBase
+	{
+		public override IEnumerable<string> SkipQueryStringParams => new[]
+		{
+			"advance_time",
+			"end",
+			"start",
+			"calc_interim",
+		};
+	}
+}

src/CodeGeneration/ApiGenerator/Overrides/Descriptors/GetAnomalyRecordsDescriptorOverrides.cs

@@ -0,0 +1,21 @@
+using System.Collections.Generic;
+using System.Linq;
+using ApiGenerator.Domain;
+
+namespace ApiGenerator.Overrides.Descriptors
+{
+	public class GetAnomalyRecordsDescriptorOverrides : DescriptorOverridesBase
+	{
+		public override IEnumerable<string> SkipQueryStringParams => new[]
+		{
+			"exclude_interim",
+			"from",
+			"size",
+			"start",
+			"end",
+			"record_score",
+			"sort",
+			"desc"
+		};
+	}
+}

src/CodeGeneration/ApiGenerator/Overrides/Descriptors/GetBucketsDescriptorOverrides.cs

@@ -0,0 +1,22 @@
+using System.Collections.Generic;
+using System.Linq;
+using ApiGenerator.Domain;
+
+namespace ApiGenerator.Overrides.Descriptors
+{
+	public class GetBucketsDescriptorOverrides : DescriptorOverridesBase
+	{
+		public override IEnumerable<string> SkipQueryStringParams => new[]
+		{
+			"expand",
+			"exclude_interim",
+			"from",
+			"size",
+			"start",
+			"end",
+			"anomaly_score",
+			"sort",
+			"desc"
+		};
+	}
+}

src/CodeGeneration/ApiGenerator/Overrides/Descriptors/GetCategoriesDescriptorOverrides.cs

@@ -0,0 +1,15 @@
+using System.Collections.Generic;
+using System.Linq;
+using ApiGenerator.Domain;
+
+namespace ApiGenerator.Overrides.Descriptors
+{
+	public class GetCategoriesDescriptorOverrides : DescriptorOverridesBase
+	{
+		public override IEnumerable<string> SkipQueryStringParams => new[]
+		{
+			"from",
+			"size"
+		};
+	}
+}

src/CodeGeneration/ApiGenerator/Overrides/Descriptors/GetInfluencersDescriptorOverrides.cs

@@ -0,0 +1,21 @@
+using System.Collections.Generic;
+using System.Linq;
+using ApiGenerator.Domain;
+
+namespace ApiGenerator.Overrides.Descriptors
+{
+	public class GetInfluencersDescriptorOverrides : DescriptorOverridesBase
+	{
+		public override IEnumerable<string> SkipQueryStringParams => new[]
+		{
+			"exclude_interim",
+			"from",
+			"size",
+			"start",
+			"end",
+			"influencer_score",
+			"sort",
+			"desc"
+		};
+	}
+}

src/CodeGeneration/ApiGenerator/Overrides/Descriptors/GetModelSnapshotsDescriptorOverrides.cs

@@ -0,0 +1,19 @@
+using System.Collections.Generic;
+using System.Linq;
+using ApiGenerator.Domain;
+
+namespace ApiGenerator.Overrides.Descriptors
+{
+	public class GetModelSnapshotsDescriptorOverrides : DescriptorOverridesBase
+	{
+		public override IEnumerable<string> SkipQueryStringParams => new[]
+		{
+			"from",
+			"size",
+			"start",
+			"end",
+			"sort",
+			"desc"
+		};
+	}
+}

src/CodeGeneration/ApiGenerator/Overrides/Descriptors/PostJobDataDescriptorOverrides.cs

@@ -0,0 +1,16 @@
+using System.Collections.Generic;
+using System.Linq;
+using ApiGenerator.Domain;
+
+namespace ApiGenerator.Overrides.Descriptors
+{
+	public class PostJobDataDescriptorOverrides : DescriptorOverridesBase
+	{
+		public override CsharpMethod PatchMethod(CsharpMethod method)
+		{
+			method.Url.Params["reset_start"].Type = "date";
+			method.Url.Params["reset_end"].Type = "date";
+			return method;
+		}
+	}
+}

src/CodeGeneration/ApiGenerator/Overrides/Descriptors/RevertModelSnapshotDescriptorOverrides.cs

@@ -0,0 +1,14 @@
+using System.Collections.Generic;
+using System.Linq;
+using ApiGenerator.Domain;
+
+namespace ApiGenerator.Overrides.Descriptors
+{
+	public class RevertModelSnapshotDescriptorOverrides : DescriptorOverridesBase
+	{
+		public override IEnumerable<string> SkipQueryStringParams => new[]
+		{
+			"delete_intervening_results"
+		};
+	}
+}

src/CodeGeneration/ApiGenerator/Overrides/Descriptors/StartDatafeedDescriptorOverrides.cs

@@ -0,0 +1,14 @@
+using System.Collections.Generic;
+
+namespace ApiGenerator.Overrides.Descriptors
+{
+	public class StartDatafeedDescriptorOverrides : DescriptorOverridesBase
+	{
+		public override IEnumerable<string> SkipQueryStringParams => new[]
+		{
+			"timeout",
+			"start",
+			"end",
+		};
+	}
+}

src/CodeGeneration/ApiGenerator/Overrides/Descriptors/StopDatafeedDescriptorOverrides.cs

@@ -0,0 +1,13 @@
+using System.Collections.Generic;
+
+namespace ApiGenerator.Overrides.Descriptors
+{
+	public class StopDatafeedDescriptorOverrides : DescriptorOverridesBase
+	{
+		public override IEnumerable<string> SkipQueryStringParams => new[]
+		{
+			"timeout",
+			"force"
+		};
+	}
+}

src/CodeGeneration/ApiGenerator/Overrides/Descriptors/UpdateModelSnapshotDescriptorOverrides.cs

@@ -0,0 +1,13 @@
+using System.Collections.Generic;
+
+namespace ApiGenerator.Overrides.Descriptors
+{
+	public class UpdateModelSnapshotDescriptorOverrides : DescriptorOverridesBase
+	{
+		public override IEnumerable<string> SkipQueryStringParams => new[]
+		{
+			"description",
+			"retain"
+		};
+	}
+}

src/CodeGeneration/ApiGenerator/Program.cs

@@ -43,7 +43,7 @@ static void Main(string[] args)
 			if (redownloadCoreSpecification)
 				RestSpecDownloader.Download(downloadBranch);
 
-			ApiGenerator.Generate(downloadBranch, "Core", "Graph", "License", "Security", "Watcher", "Info");
+			ApiGenerator.Generate(downloadBranch, "Core", "Graph", "License", "Security", "Watcher", "Info", "MachineLearning");
 
 			//ApiGenerator.Generate(); //generates everything under ApiSpecification
 		}