Retention progress

MarkMpn · Jun 26, 2023 · 72e820f · 72e820f
1 parent da6c9f3
commit 72e820f
Show file tree

Hide file tree

Showing 16 changed files with 99 additions and 38 deletions.
diff --git a/MarkMpn.Sql4Cds.Engine.NetCore/MarkMpn.Sql4Cds.Engine.NetCore.csproj b/MarkMpn.Sql4Cds.Engine.NetCore/MarkMpn.Sql4Cds.Engine.NetCore.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <AssemblyName>MarkMpn.Sql4Cds.Engine</AssemblyName>
     <RootNamespace>MarkMpn.Sql4Cds.Engine</RootNamespace>
   </PropertyGroup>
@@ -10,8 +10,8 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.ApplicationInsights" Version="2.20.0" />
-    <PackageReference Include="Microsoft.PowerPlatform.Dataverse.Client" Version="0.6.6" />
-    <PackageReference Include="Microsoft.SqlServer.TransactSql.ScriptDom" Version="150.4897.1" />
+    <PackageReference Include="Microsoft.PowerPlatform.Dataverse.Client" Version="1.1.9" />
+    <PackageReference Include="Microsoft.SqlServer.TransactSql.ScriptDom" Version="161.8834.0" />
     <PackageReference Include="System.Data.SqlClient" Version="4.8.3" />
     <PackageReference Include="XPath2.Extensions" Version="1.1.3" />
   </ItemGroup>

diff --git a/MarkMpn.Sql4Cds.Engine.NetFx/MarkMpn.Sql4Cds.Engine.NetFx.csproj b/MarkMpn.Sql4Cds.Engine.NetFx/MarkMpn.Sql4Cds.Engine.NetFx.csproj
@@ -64,16 +64,19 @@
     <PackageReference Include="Microsoft.ApplicationInsights">
       <Version>2.21.0</Version>
     </PackageReference>
+    <PackageReference Include="Microsoft.CrmSdk.CoreAssemblies">
+      <Version>9.0.2.49</Version>
+    </PackageReference>
     <PackageReference Include="Microsoft.CrmSdk.XrmTooling.CoreAssembly">
-      <Version>9.1.1.1</Version>
+      <Version>9.1.1.32</Version>
     </PackageReference>
     <PackageReference Include="Microsoft.SourceLink.GitHub">
       <Version>1.1.1</Version>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
     <PackageReference Include="Microsoft.SqlServer.TransactSql.ScriptDom">
-      <Version>150.4897.1</Version>
+      <Version>161.8834.0</Version>
     </PackageReference>
     <PackageReference Include="XPath2.Extensions">
       <Version>1.1.3</Version>

diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDataNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDataNode.cs
@@ -495,7 +495,7 @@ private bool TranslateFetchXMLCriteria(NodeCompilationContext context, IAttribut
                         entityname = StandardizeAlias(entityAlias, targetEntityAlias, items),
                         attribute = RemoveAttributeAlias(attrName.ToLowerInvariant(), entityAlias, targetEntityAlias, items),
                         @operator = op,
-                        valueof = RemoveAttributeAlias(attrName2.ToLowerInvariant(), entityAlias, targetEntityAlias, items)
+                        ValueOf = RemoveAttributeAlias(attrName2.ToLowerInvariant(), entityAlias, targetEntityAlias, items)
                     };
                     return true;
                 }

diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs
@@ -281,7 +281,23 @@ protected override IEnumerable<Entity> ExecuteInternal(NodeExecutionContext cont
             if (BypassCustomPluginExecution)
                 req.Parameters["BypassCustomPluginExecution"] = true;
 
-            var res = ((RetrieveMultipleResponse)dataSource.Execute(req)).EntityCollection;
+            EntityCollection res;
+
+            try
+            {
+                res = ((RetrieveMultipleResponse)dataSource.Execute(req)).EntityCollection;
+            }
+            catch (FaultException<OrganizationServiceFault> ex)
+            {
+                // Archive queries can fail with this error code if the Synapse database isn't provisioned yet or
+                // no retention policy has yet been applied to this table. In either case there are no records to return
+                // so we can just return an empty result set rather than erroring
+                if (FetchXml.DataSource == "archive" && (ex.Detail.ErrorCode == -2146863832 || ex.Detail.ErrorCode == -2146863829))
+                    yield break;
+
+                throw;
+            }
+
             PagesRetrieved++;
 
             var count = res.Entities.Count;
@@ -1214,7 +1230,7 @@ private void ConvertQueryHints(IList<OptimizerHint> hints)
                 .Where(hint => hint != null));
 
             if (!String.IsNullOrEmpty(options))
-                FetchXml.options = options;
+                FetchXml.Options = options;
         }
 
         private void ApplyPageSizeHint(IList<OptimizerHint> hints)

diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FoldableJoinNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FoldableJoinNode.cs
@@ -257,6 +257,10 @@ private bool FoldFetchXmlJoin(NodeCompilationContext context, IList<OptimizerHin
             if (!leftFetch.DataSource.Equals(rightFetch.DataSource, StringComparison.OrdinalIgnoreCase))
                 return false;
 
+            // Can't join with archived data
+            if (leftFetch.FetchXml.DataSource == "archive" || rightFetch.FetchXml.DataSource == "archive")
+                return false;
+
             // If one source is distinct and the other isn't, joining the two won't produce the expected results
             if (leftFetch.FetchXml.distinct ^ rightFetch.FetchXml.distinct)
                 return false;

diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlanBuilder.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlanBuilder.cs
@@ -3421,7 +3421,9 @@ private IDataExecutionPlanNodeInternal ConvertTableReference(TableReference refe
                     throw new NotSupportedQueryFragmentException("Unknown table name", table);
                 }
 
-                if (!String.IsNullOrEmpty(table.SchemaObject.SchemaIdentifier?.Value) && !table.SchemaObject.SchemaIdentifier.Value.Equals("dbo", StringComparison.OrdinalIgnoreCase))
+                if (!String.IsNullOrEmpty(table.SchemaObject.SchemaIdentifier?.Value) &&
+                    !table.SchemaObject.SchemaIdentifier.Value.Equals("dbo", StringComparison.OrdinalIgnoreCase) &&
+                    !table.SchemaObject.SchemaIdentifier.Value.Equals("archive", StringComparison.OrdinalIgnoreCase))
                     throw new NotSupportedQueryFragmentException("Unknown table name", table);
 
                 // Validate the entity name
@@ -3447,7 +3449,7 @@ private IDataExecutionPlanNodeInternal ConvertTableReference(TableReference refe
                     throw new NotSupportedQueryFragmentException("Unsupported temporal clause", table.TemporalClause);
 
                 // Convert to a simple FetchXML source
-                return new FetchXmlScan
+                var fetchXmlScan = new FetchXmlScan
                 {
                     DataSource = dataSource.Name,
                     FetchXml = new FetchXml.FetchType
@@ -3464,6 +3466,17 @@ private IDataExecutionPlanNodeInternal ConvertTableReference(TableReference refe
                     Alias = table.Alias?.Value ?? entityName,
                     ReturnFullSchema = true
                 };
+
+                // Check if this should be using the long-term retention table
+                if (table.SchemaObject.SchemaIdentifier?.Value.Equals("archive", StringComparison.OrdinalIgnoreCase) == true)
+                {
+                    if (meta.IsRetentionEnabled != true)
+                        throw new NotSupportedQueryFragmentException("Unknown table name", table) { Suggestion = "Ensure long term retention is enabled for this table - see https://learn.microsoft.com/en-us/power-apps/maker/data-platform/data-retention-set?WT.mc_id=DX-MVP-5004203" };
+
+                    fetchXmlScan.FetchXml.DataSource = "archive";
+                }
+
+                return fetchXmlScan;
             }
 
             if (reference is QualifiedJoin join)

diff --git a/MarkMpn.Sql4Cds.Engine/FetchXml.Extensions.cs b/MarkMpn.Sql4Cds.Engine/FetchXml.Extensions.cs
@@ -0,0 +1,43 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Text;
+using System.Xml.Serialization;
+
+namespace MarkMpn.Sql4Cds.Engine.FetchXml
+{
+    partial class condition
+    {
+        [XmlAttribute("valueof")]
+        public string ValueOf { get; set; }
+
+        [XmlAttribute(Namespace = "MarkMpn.SQL4CDS")]
+        [DefaultValue(false)]
+        public bool IsVariable { get; set; }
+    }
+
+    partial class conditionValue
+    {
+        [XmlAttribute(Namespace = "MarkMpn.SQL4CDS")]
+        [DefaultValue(false)]
+        public bool IsVariable { get; set; }
+    }
+
+    partial class FetchLinkEntityType
+    {
+        [XmlIgnore]
+        public bool SemiJoin { get; set; }
+
+        [XmlIgnore]
+        public bool RequireTablePrefix { get; set; }
+    }
+
+    partial class FetchType
+    {
+        [XmlAttribute("options")]
+        public string Options { get; set; }
+
+        [XmlAttribute("datasource")]
+        public string DataSource { get; set; }
+    }
+}
diff --git a/MarkMpn.Sql4Cds.Engine/FetchXml.cs b/MarkMpn.Sql4Cds.Engine/FetchXml.cs
@@ -12,7 +12,6 @@
 // This source code was auto-generated by xsd, Version=4.6.1055.0.
 // 
 namespace MarkMpn.Sql4Cds.Engine.FetchXml {
-    using System.ComponentModel;
     using System.Xml.Serialization;
 
 
@@ -219,13 +218,6 @@ public bool uihiddenSpecified {
                 this.uihiddenFieldSpecified = value;
             }
         }
-
-        [XmlAttribute]
-        public string valueof { get; set; }
-
-        [XmlAttribute(Namespace = "MarkMpn.SQL4CDS")]
-        [DefaultValue(false)]
-        public bool IsVariable { get; set; }
     }
 
     /// <remarks/>
@@ -274,10 +266,6 @@ public string Value {
                 this.valueField = value;
             }
         }
-
-        [XmlAttribute(Namespace = "MarkMpn.SQL4CDS")]
-        [DefaultValue(false)]
-        public bool IsVariable { get; set; }
     }
 
     /// <remarks/>
@@ -588,12 +576,6 @@ public string prefilterparametername {
                 this.prefilterparameternameField = value;
             }
         }
-
-        [XmlIgnore]
-        public bool SemiJoin { get; set; }
-
-        [XmlIgnore]
-        public bool RequireTablePrefix { get; set; }
     }
 
     /// <remarks/>
@@ -1699,9 +1681,6 @@ public bool nolock {
                 this.nolockField = value;
             }
         }
-
-        [System.Xml.Serialization.XmlAttributeAttribute("options")]
-        public string options { get; set; }
     }
 
     /// <remarks/>

diff --git a/MarkMpn.Sql4Cds.Engine/FetchXml2Sql.cs b/MarkMpn.Sql4Cds.Engine/FetchXml2Sql.cs
@@ -652,7 +652,7 @@ private static BooleanExpression GetCondition(IOrganizationService org, IAttribu
 
             // Get the literal value to compare to
             object parameterValue = null;
-            if (!String.IsNullOrEmpty(condition.valueof))
+            if (!String.IsNullOrEmpty(condition.ValueOf))
             {
                 value = new ColumnReferenceExpression
                 {

diff --git a/MarkMpn.Sql4Cds.Engine/MarkMpn.Sql4Cds.Engine.FetchXml.projitems b/MarkMpn.Sql4Cds.Engine/MarkMpn.Sql4Cds.Engine.FetchXml.projitems
@@ -10,6 +10,7 @@
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="$(MSBuildThisFileDirectory)DatabaseIdentifiers.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)FetchXml.Extensions.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)ScriptDom.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)AttributeMetadataCache.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)DisconnectedException.cs" />

diff --git a/MarkMpn.Sql4Cds.Engine/MarkMpn.Sql4Cds.Engine.projitems b/MarkMpn.Sql4Cds.Engine/MarkMpn.Sql4Cds.Engine.projitems
@@ -31,6 +31,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)ExecutionPlan\SystemFunctionNode.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)ExecutionPlan\XmlWriterNode.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)ExecutionPlan\XPath2ExpressionContext.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)FetchXml.Extensions.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)NodeContext.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)DatabaseIdentifiers.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)DataSource.cs" />

diff --git a/MarkMpn.Sql4Cds.SSMS.18/MarkMpn.Sql4Cds.SSMS.18.csproj b/MarkMpn.Sql4Cds.SSMS.18/MarkMpn.Sql4Cds.SSMS.18.csproj
@@ -169,7 +169,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.CrmSdk.XrmTooling.CoreAssembly">
-      <Version>9.1.1.1</Version>
+      <Version>9.1.1.32</Version>
     </PackageReference>
     <PackageReference Include="Microsoft.VisualStudio.Imaging.Interop.14.0.DesignTime">
       <Version>14.3.25408</Version>

diff --git a/MarkMpn.Sql4Cds.SSMS.19/MarkMpn.Sql4Cds.SSMS.19.csproj b/MarkMpn.Sql4Cds.SSMS.19/MarkMpn.Sql4Cds.SSMS.19.csproj
@@ -162,7 +162,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.CrmSdk.XrmTooling.CoreAssembly">
-      <Version>9.1.1.1</Version>
+      <Version>9.1.1.32</Version>
     </PackageReference>
     <PackageReference Include="Microsoft.Data.SqlClient">
       <Version>3.1.1</Version>

diff --git a/MarkMpn.Sql4Cds.Tests/MarkMpn.Sql4Cds.Tests.csproj b/MarkMpn.Sql4Cds.Tests/MarkMpn.Sql4Cds.Tests.csproj
@@ -99,7 +99,7 @@
       <Version>1.58.1</Version>
     </PackageReference>
     <PackageReference Include="Microsoft.CrmSdk.XrmTooling.CoreAssembly">
-      <Version>9.1.1.1</Version>
+      <Version>9.1.1.32</Version>
     </PackageReference>
     <PackageReference Include="MSTest.TestAdapter">
       <Version>3.0.2</Version>

diff --git a/MarkMpn.Sql4Cds/ILMergeConfig.json b/MarkMpn.Sql4Cds/ILMergeConfig.json
@@ -12,6 +12,7 @@
   },
   "Advanced": {
     "AttributeFile": "$(TargetDir)\\MarkMpn.Sql4Cds.dll",
-    "CopyAttributes": false
+    "CopyAttributes": false,
+    "DebugInfo": true
   }
 }
diff --git a/MarkMpn.Sql4Cds/MarkMpn.Sql4Cds.csproj b/MarkMpn.Sql4Cds/MarkMpn.Sql4Cds.csproj
@@ -259,7 +259,7 @@
       <Version>2.21.0</Version>
     </PackageReference>
     <PackageReference Include="Microsoft.SqlServer.TransactSql.ScriptDom">
-      <Version>150.4897.1</Version>
+      <Version>161.8834.0</Version>
     </PackageReference>
     <PackageReference Include="System.Runtime">
       <Version>4.3.1</Version>
@@ -268,7 +268,7 @@
       <Version>2.2.10</Version>
     </PackageReference>
     <PackageReference Include="XrmToolBoxPackage">
-      <Version>1.2022.10.58</Version>
+      <Version>1.2023.6.65</Version>
     </PackageReference>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />