Improved template and added error handling for Cosmos

Mark-RSK · Oct 7, 2018 · f95e267 · f95e267
1 parent e076e5e
commit f95e267
Show file tree

Hide file tree

Showing 20 changed files with 322 additions and 52 deletions.
diff --git a/Exemplars/StandardFunctions/CosmosExample.cs b/Exemplars/StandardFunctions/CosmosExample.cs
@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Net.Http.Formatting;
@@ -22,20 +23,29 @@ public static void Run([CosmosDBTrigger(
             ConnectionStringSetting = "cosmosConnectionString",
             LeaseCollectionName = "leases")]IReadOnlyList<Document> input, ILogger log)
         {
+            //Lazy<string> errorHandler = new Lazy<string>(() => string.Empty);
             foreach (Document document in input)
             {
-                document.GetPropertyValue<string>("a");
-                using (MemoryStream memoryStream = new MemoryStream())
+                try
                 {
-                    document.SaveTo(memoryStream, SerializationFormattingPolicy.None, new JsonSerializerSettings
+
+
+                    document.GetPropertyValue<string>("a");
+                    using (MemoryStream memoryStream = new MemoryStream())
                     {
-                        ContractResolver = new CamelCasePropertyNamesContractResolver()
-                    });
-                    string json = Encoding.UTF8.GetString(memoryStream.ToArray());
-                    string myCommand = JsonConvert.DeserializeObject<string>(json);
-                }                    
+                        document.SaveTo(memoryStream, SerializationFormattingPolicy.None, new JsonSerializerSettings
+                        {
+                            ContractResolver = new CamelCasePropertyNamesContractResolver()
+                        });
+                        string json = Encoding.UTF8.GetString(memoryStream.ToArray());
+                        string myCommand = JsonConvert.DeserializeObject<string>(json);
+                    }
+                }
+                catch
+                {
+                    break;
+                }
             }
-
             if (input != null && input.Count > 0)
             {
                 log.LogInformation("Documents modified " + input.Count);

diff --git a/Samples/Scratch/SwaggerBuildOut/ExampleCosmosErrorHandler.cs b/Samples/Scratch/SwaggerBuildOut/ExampleCosmosErrorHandler.cs
@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading.Tasks;
+using FunctionMonkey.Commanding.Cosmos.Abstractions;
+using Microsoft.Azure.Documents;
+using Microsoft.Build.Framework;
+using Microsoft.Extensions.Logging;
+
+namespace SwaggerBuildOut
+{
+    public class ExampleCosmosErrorHandler : ICosmosDbErrorHandler
+    {
+        private readonly ILogger<ExampleCosmosErrorHandler> _logger;
+
+        public ExampleCosmosErrorHandler(ILogger<ExampleCosmosErrorHandler> logger)
+        {
+            _logger = logger;
+        }
+
+        public Task<bool> HandleError(Exception ex, Document document)
+        {
+            _logger.LogError(ex, "Went wrong");
+            return Task.FromResult(true);
+        }
+    }
+}
diff --git a/Samples/Scratch/SwaggerBuildOut/FunctionAppConfiguration.cs b/Samples/Scratch/SwaggerBuildOut/FunctionAppConfiguration.cs
@@ -25,13 +25,15 @@ public void Build(IFunctionHostBuilder builder)
                     commandRegistry.Register<CosmosCommandHandler>();
                     commandRegistry.Register<CosmosDocumentCommandHandler>();
                     commandRegistry.Register<CosmosDocumentBatchCommandHandler>();
+
+                    serviceCollection.AddLogging();
                 })
                 .OpenApiEndpoint(openApi => openApi
                     .Title("A Simple API")
                     .Version("0.0.0")
                     .UserInterface()
                 )
-                .OutputAuthoredSource(@"d:\wip\scratch\outputSource")
+                .OutputAuthoredSource(@"c:\wip\scratch\outputSource")
                 .Functions(functions => functions
                     .HttpRoute("/HelloWorld", route => route
                         .HttpFunction<HelloWorldCommand>("/{name}", AuthorizationTypeEnum.Anonymous, HttpMethod.Get)    
@@ -41,9 +43,9 @@ public void Build(IFunctionHostBuilder builder)
                     )
                     .OpenApiDescription("A route description")
                     .CosmosDb("CosmosConnection", cosmos => cosmos
-                        .ChangeFeedFunction<CosmosCommand>("Items", "ToDoList", leaseCollectionPrefix:"fn1")//, convertToPascalCase:true)
+                        .ChangeFeedFunction<CosmosCommand, ExampleCosmosErrorHandler>("Items", "ToDoList", leaseCollectionPrefix:"fn1")//, convertToPascalCase:true)
                         //.ChangeFeedFunction<CosmosDocumentCommand>("Items", "ToDoList")
-                        .ChangeFeedFunction<CosmosDocumentBatchCommand>("Items", "ToDoList", leaseCollectionPrefix:"fn2")
+                        //.ChangeFeedFunction<CosmosDocumentBatchCommand>("Items", "ToDoList", leaseCollectionPrefix:"fn2")
                     )
                     /*.HttpRoute("/Add", route => route
                         .HttpFunction<AddCommand>(AuthorizationTypeEnum.Anonymous,HttpMethod.Post)

diff --git a/Samples/Scratch/SwaggerBuildOut/Handlers/CosmosCommandHandler.cs b/Samples/Scratch/SwaggerBuildOut/Handlers/CosmosCommandHandler.cs
@@ -1,4 +1,5 @@
-using System.Threading.Tasks;
+using System;
+using System.Threading.Tasks;
 using AzureFromTheTrenches.Commanding.Abstractions;
 using SwaggerBuildOut.Commands;
 
@@ -8,6 +9,7 @@ class CosmosCommandHandler : ICommandHandler<CosmosCommand>
     {
         public Task ExecuteAsync(CosmosCommand command)
         {
+            throw new Exception("eeek");
             return Task.CompletedTask;
         }
     }

diff --git a/Source/FunctionMonkey.Abstractions/Builders/ICosmosDbFunctionBuilder.cs b/Source/FunctionMonkey.Abstractions/Builders/ICosmosDbFunctionBuilder.cs
@@ -1,4 +1,5 @@
 using AzureFromTheTrenches.Commanding.Abstractions;
+using FunctionMonkey.Commanding.Cosmos.Abstractions;
 
 namespace FunctionMonkey.Abstractions.Builders
 {
@@ -52,5 +53,55 @@ ICosmosDbFunctionBuilder ChangeFeedFunction<TCommand>(string collectionName,
             int? leasesCollectionThroughput=null
             )
             where TCommand : ICommand;
+
+        ///  <summary>
+        ///  Associate a function with the specified collection and database
+        ///  </summary>
+        ///  <typeparam name="TCommand"></typeparam>
+        /// <typeparam name="TCosmosDbErrorHandler">A type that provides error handling for document processing</typeparam>
+        /// <param name="collectionName">The Cosmos collection name</param>
+        ///  <param name="databaseName">The Cosmos database name</param>
+        ///  <param name="leaseCollectionName">The cosmos collection to use for leases - defaults to leases</param>
+        ///  <param name="leaseDatabaseName">The cosmos database to use for leases - defaults to the database name</param>
+        ///  <param name="createLeaseCollectionIfNotExists">Creates the lease collection if it doesn't exist, defaults to false</param>
+        ///  <param name="startFromBeginning">Start from the beginning of the change feed, defaults to false</param>
+        ///  <param name="convertToPascalCase">
+        ///  Is the Cosmos document in camel case. Defaults to true in which case C# property names will be converted to camel case. Note that
+        ///  due to limitations in the current Functions / Cosmos Change Feed Processor this involves serializing to a memory stream and then
+        ///  deserializing.
+        /// 
+        ///  If this limitation continues then I will seek to optimize this by compiling camel case model classes and mapping them to pascal
+        ///  case C# classes via Roslyn.
+        /// 
+        ///  Alternatively you can use the JsonProperty attribute on all your properties. Messy I know.
+        /// 
+        ///  Due to the potential expense this defaults to false.
+        ///  </param>
+        /// <param name="leaseCollectionPrefix">When set, it adds a prefix to the leases created in the Lease collection for this Function</param>
+        /// <param name="maxItemsPerInvocation">When set, it customizes the maximum amount of items received per Function call.</param>
+        /// <param name="feedPollDelay">When set, it defines, in milliseconds, the delay in between polling a partition for new changes on the feed, after all current changes are drained. Default is 5000 (5 seconds).</param>
+        /// <param name="leaseAcquireInterval">When set, it defines, in milliseconds, the interval to kick off a task to compute if partitions are distributed evenly among known host instances. Default is 13000 (13 seconds).</param>
+        /// <param name="leaseExpirationInterval">When set, it defines, in milliseconds, the interval for which the lease is taken on a lease representing a partition. If the lease is not renewed within this interval, it will cause it to expire and ownership of the partition will move to another instance. Default is 60000 (60 seconds)</param>
+        /// <param name="leaseRenewInterval">When set, it defines, in milliseconds, the renew interval for all leases for partitions currently held by an instance. Default is 17000 (17 seconds).</param>
+        /// <param name="checkpointFrequency">When set, it defines, in milliseconds, the interval between lease checkpoints. Default is always after a successful Function call.</param>
+        /// <param name="leasesCollectionThroughput">Defines the amount of Request Units to assign when the leases collection is created. This setting is only used When createLeaseCollectionIfNotExists is set to true. This parameter is automatically set when the binding is created using the portal.</param>
+        /// <returns></returns>
+        ICosmosDbFunctionBuilder ChangeFeedFunction<TCommand, TCosmosDbErrorHandler>(string collectionName,
+            string databaseName,
+            string leaseCollectionName = "leases",
+            string leaseDatabaseName = null,
+            bool createLeaseCollectionIfNotExists = false,
+            bool startFromBeginning = false,
+            bool convertToPascalCase = false,
+            string leaseCollectionPrefix = null,
+            int? maxItemsPerInvocation = null,
+            int? feedPollDelay = null,
+            int? leaseAcquireInterval = null,
+            int? leaseExpirationInterval = null,
+            int? leaseRenewInterval = null,
+            int? checkpointFrequency = null,
+            int? leasesCollectionThroughput = null
+        )
+            where TCommand : ICommand where TCosmosDbErrorHandler : ICosmosDbErrorHandler;
     }
 }
diff --git a/Source/FunctionMonkey.Abstractions/FunctionMonkey.Abstractions.csproj b/Source/FunctionMonkey.Abstractions/FunctionMonkey.Abstractions.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
-    <Version>0.16.0-beta000</Version>
+    <Version>0.16.1-beta000</Version>
     <Authors>James Randall</Authors>
     <PackageLicenseUrl>https://raw.githubusercontent.com/JamesRandall/AzureFromTheTrenches.Commanding/master/LICENSE</PackageLicenseUrl>
     <PackageProjectUrl>https://commanding.azurefromthetrenches.com/</PackageProjectUrl>
@@ -12,6 +12,7 @@
   <ItemGroup>
     <PackageReference Include="AzureFromTheTrenches.Commanding.Abstractions" Version="8.1.1" />
     <PackageReference Include="Microsoft.AspNetCore.Mvc.Abstractions" Version="2.1.1" />
+    <PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="2.0.0" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="2.1.1" />
   </ItemGroup>
 

diff --git a/Source/FunctionMonkey.Abstractions/ICosmosDbErrorHandler.cs b/Source/FunctionMonkey.Abstractions/ICosmosDbErrorHandler.cs
@@ -0,0 +1,21 @@
+using System;
+using System.Threading.Tasks;
+using Microsoft.Azure.Documents;
+
+namespace FunctionMonkey.Commanding.Cosmos.Abstractions
+{
+    /// <summary>
+    /// Implementations of this handle Cosmos errors at the Function level.
+    /// </summary>
+    public interface ICosmosDbErrorHandler
+    {
+        /// <summary>
+        /// Handle a error processing a Cosmos document.
+        /// To abort processing of the current batch return false.
+        /// </summary>
+        /// <param name="ex">The exception that caused the error</param>
+        /// <param name="document">The document that caused the error</param>
+        /// <returns>True to continue processing, false to stop processing of the batch</returns>
+        Task<bool> HandleError(Exception ex, Document document);
+    }
+}
diff --git a/Source/FunctionMonkey.Commanding.Abstractions/FunctionMonkey.Commanding.Abstractions.csproj b/Source/FunctionMonkey.Commanding.Abstractions/FunctionMonkey.Commanding.Abstractions.csproj
@@ -2,14 +2,14 @@
 
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
-    <Version>0.16.0-beta000</Version>
+    <Version>0.16.1-beta000</Version>
     <Company>James Randall</Company>
     <Authors>James Randall</Authors>
     <PackageLicenseUrl>https://raw.githubusercontent.com/JamesRandall/AzureFromTheTrenches.Commanding/master/LICENSE</PackageLicenseUrl>
     <PackageProjectUrl>https://commanding.azurefromthetrenches.com/</PackageProjectUrl>
     <RepositoryUrl>https://github.com/JamesRandall/FunctionMonkey.git</RepositoryUrl>
-    <AssemblyVersion>0.16.0.0</AssemblyVersion>
-    <FileVersion>0.16.0.0</FileVersion>
+    <AssemblyVersion>0.16.1.0</AssemblyVersion>
+    <FileVersion>0.16.1.0</FileVersion>
   </PropertyGroup>
 
 </Project>
diff --git a/...onkey.Commanding.Cosmos.Abstractions/FunctionMonkey.Commanding.Cosmos.Abstractions.csproj b/...onkey.Commanding.Cosmos.Abstractions/FunctionMonkey.Commanding.Cosmos.Abstractions.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
-    <Version>0.16.0-beta000</Version>
+    <Version>0.16.1-beta000</Version>
   </PropertyGroup>
 
   <ItemGroup>

diff --git a/Source/FunctionMonkey.Compiler/FunctionMonkey.Compiler.csproj b/Source/FunctionMonkey.Compiler/FunctionMonkey.Compiler.csproj
@@ -3,7 +3,7 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFramework>netcoreapp2.0</TargetFramework>    
-    <Version>0.16.0-beta000</Version>
+    <Version>0.16.1-beta000</Version>
     <AssemblyName>FunctionMonkey.Compiler</AssemblyName>
     <PackageId>FunctionMonkey.Compiler</PackageId>
   </PropertyGroup>
@@ -358,8 +358,8 @@
     <IntermediatePackDir>$(MSBuildProjectDirectory)/bin/$(Configuration)/publish/</IntermediatePackDir>
     <PublishDir>$(IntermediatePackDir)$(TargetFramework)/</PublishDir>
     <NuspecProperties>publishDir=$([MSBuild]::NormalizeDirectory($(IntermediatePackDir)))</NuspecProperties>
-    <AssemblyVersion>0.16.0.0</AssemblyVersion>
-    <FileVersion>0.16.0.0</FileVersion>
+    <AssemblyVersion>0.16.1.0</AssemblyVersion>
+    <FileVersion>0.16.1.0</FileVersion>
     <PackageReleaseNotes></PackageReleaseNotes>
   </PropertyGroup>
 

diff --git a/Source/FunctionMonkey.Compiler/FunctionMonkey.Compiler.nuspec b/Source/FunctionMonkey.Compiler/FunctionMonkey.Compiler.nuspec
@@ -2,7 +2,7 @@
 <package xmlns="http://schemas.microsoft.com/packaging/2012/06/nuspec.xsd">
   <metadata>
     <id>FunctionMonkey.Compiler</id>
-    <version>0.16.0-beta000</version>
+    <version>0.16.1-beta000</version>
     <authors>James Randall</authors>
     <description>Generates Azure Functions from command registrations</description>
     <licenseUrl>https://raw.githubusercontent.com/JamesRandall/AzureFromTheTrenches.Commanding/master/LICENSE</licenseUrl>

diff --git a/Source/FunctionMonkey.Compiler/Templates/cosmosdb.csharp.handlebars b/Source/FunctionMonkey.Compiler/Templates/cosmosdb.csharp.handlebars
@@ -41,33 +41,46 @@ namespace {{Namespace}}
                 var result = await FunctionMonkey.Runtime.CommandDispatcher.DispatchAsync(command);
                 // The result will be useful when we want to include outputs too
             {{else}}
+                {{#if ErrorHandlerTypeName}}
+                Lazy<{{ErrorHandlerTypeName}}> errorHandler = new Lazy<{{ErrorHandlerTypeName}}>(
+                    () => ({{ErrorHandlerTypeName}})FunctionMonkey.Runtime.ServiceProvider.GetService(typeof({{ErrorHandlerTypeName}}))                                                                               
+                );
+                {{/if}}
                 foreach(Document document in input)
                 {
-                    {{CommandTypeName}} command = null;
-                    {{#if IsDocumentCommand}}
-                        command = new {{CommandTypeName}}();
-                        command.Document = document;
-                    {{else}}
-                        {{#if ConvertToPascalCase}}
-                            using (MemoryStream memoryStream = new MemoryStream())
-                            {
-                                document.SaveTo(memoryStream, SerializationFormattingPolicy.None, new JsonSerializerSettings
-                                {
-                                    ContractResolver = new CamelCasePropertyNamesContractResolver()
-                                });
-                                string json = Encoding.UTF8.GetString(memoryStream.ToArray());
-                                command = JsonConvert.DeserializeObject<{{CommandTypeName}}>(json);
-                            }    
-                        {{else}}
+                    try
+                    {
+                        {{CommandTypeName}} command = null;
+                        {{#if IsDocumentCommand}}
                             command = new {{CommandTypeName}}();
-                            {{#each CommandProperties}}
-                                command.{{Name}} = document.GetPropertyValue<{{TypeName}}>("{{CosmosPropertyName}}");
-                            {{/each}}
+                            command.Document = document;
+                        {{else}}
+                            {{#if ConvertToPascalCase}}
+                                string json = document.ToString();
+                                command = JsonConvert.DeserializeObject<{{CommandTypeName}}>(json);  
+                            {{else}}
+                                command = new {{CommandTypeName}}();
+                                {{#each CommandProperties}}
+                                    command.{{Name}} = document.GetPropertyValue<{{TypeName}}>("{{CosmosPropertyName}}");
+                                {{/each}}
+                            {{/if}}
                         {{/if}}
-                    {{/if}}
 
-                    var result = await FunctionMonkey.Runtime.CommandDispatcher.DispatchAsync(command);
-                    // The result will be useful when we want to include outputs too
+                        var result = await FunctionMonkey.Runtime.CommandDispatcher.DispatchAsync(command);
+                        // The result will be useful when we want to include outputs too
+                    }
+                    catch(Exception ex)
+                    {
+                        {{#if ErrorHandlerTypeName}}
+                            bool shouldContinue = await errorHandler.Value.HandleError(ex, document);
+                            if (!shouldContinue)
+                            {
+                                break;
+                            }
+                        {{else}}
+                            log.LogError(ex, "Error occurred processing document with id {Id}", document.Id);
+                        {{/if}}
+                    }
                 }
             {{/if}}
         }

diff --git a/Source/FunctionMonkey.FluentValidation/FunctionMonkey.FluentValidation.csproj b/Source/FunctionMonkey.FluentValidation/FunctionMonkey.FluentValidation.csproj
@@ -2,14 +2,14 @@
 
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
-    <Version>0.16.0-beta000</Version>
+    <Version>0.16.1-beta000</Version>
     <Authors>James Randall</Authors>
     <Company>James Randall</Company>
     <PackageLicenseUrl>https://raw.githubusercontent.com/JamesRandall/AzureFromTheTrenches.Commanding/master/LICENSE</PackageLicenseUrl>
     <PackageProjectUrl>https://commanding.azurefromthetrenches.com/</PackageProjectUrl>
     <RepositoryUrl>https://github.com/JamesRandall/FunctionMonkey.git</RepositoryUrl>
-    <AssemblyVersion>0.16.0.0</AssemblyVersion>
-    <FileVersion>0.16.0.0</FileVersion>
+    <AssemblyVersion>0.16.1.0</AssemblyVersion>
+    <FileVersion>0.16.1.0</FileVersion>
   </PropertyGroup>
 
   <ItemGroup>