From 1a1c177ba37ade809a01cb1d91c9a804b51f18f1 Mon Sep 17 00:00:00 2001 From: Mark Carrington <31017244+MarkMpn@users.noreply.github.com> Date: Fri, 27 Oct 2023 11:47:21 +0100 Subject: [PATCH] Improved FetchXML to SQL conversion for hierarchical filters --- .../FakeXrmEasyTestsBase.cs | 34 ++++ .../FetchXml2SqlTests.cs | 122 ++++++++++++- .../Metadata/Account.cs | 7 + MarkMpn.Sql4Cds.Engine/FetchXml2Sql.cs | 162 +++++++++--------- 4 files changed, 247 insertions(+), 78 deletions(-) diff --git a/MarkMpn.Sql4Cds.Engine.FetchXml.Tests/FakeXrmEasyTestsBase.cs b/MarkMpn.Sql4Cds.Engine.FetchXml.Tests/FakeXrmEasyTestsBase.cs index 1f34ee35..3355415b 100644 --- a/MarkMpn.Sql4Cds.Engine.FetchXml.Tests/FakeXrmEasyTestsBase.cs +++ b/MarkMpn.Sql4Cds.Engine.FetchXml.Tests/FakeXrmEasyTestsBase.cs @@ -8,6 +8,7 @@ using FakeXrmEasy.FakeMessageExecutors; using Microsoft.Crm.Sdk.Messages; using Microsoft.Xrm.Sdk; +using Microsoft.Xrm.Sdk.Metadata; namespace MarkMpn.Sql4Cds.Engine.FetchXml.Tests { @@ -23,6 +24,39 @@ public FakeXrmEasyTestsBase() _context.AddFakeMessageExecutor(new WhoAmIHandler()); _service = _context.GetOrganizationService(); + + SetRelationships(_context); + } + + private void SetRelationships(XrmFakedContext context) + { + foreach (var entity in context.CreateMetadataQuery()) + { + if (entity.OneToManyRelationships == null) + typeof(EntityMetadata).GetProperty(nameof(EntityMetadata.OneToManyRelationships)).SetValue(entity, Array.Empty()); + + if (entity.ManyToOneRelationships == null) + typeof(EntityMetadata).GetProperty(nameof(EntityMetadata.ManyToOneRelationships)).SetValue(entity, Array.Empty()); + + if (entity.LogicalName == "account") + { + // Add parentaccountid relationship + var relationship = new OneToManyRelationshipMetadata + { + SchemaName = "account_parentaccount", + ReferencedEntity = "account", + ReferencedAttribute = "accountid", + ReferencingEntity = "account", + ReferencingAttribute = "parentaccountid", + IsHierarchical = true + }; + + typeof(EntityMetadata).GetProperty(nameof(EntityMetadata.OneToManyRelationships)).SetValue(entity, entity.OneToManyRelationships.Concat(new[] { relationship }).ToArray()); + typeof(EntityMetadata).GetProperty(nameof(EntityMetadata.ManyToOneRelationships)).SetValue(entity, entity.ManyToOneRelationships.Concat(new[] { relationship }).ToArray()); + } + + context.SetEntityMetadata(entity); + } } } diff --git a/MarkMpn.Sql4Cds.Engine.FetchXml.Tests/FetchXml2SqlTests.cs b/MarkMpn.Sql4Cds.Engine.FetchXml.Tests/FetchXml2SqlTests.cs index 2ff00c19..21cdd4ae 100644 --- a/MarkMpn.Sql4Cds.Engine.FetchXml.Tests/FetchXml2SqlTests.cs +++ b/MarkMpn.Sql4Cds.Engine.FetchXml.Tests/FetchXml2SqlTests.cs @@ -431,9 +431,129 @@ public void ArchiveJoins() Assert.AreEqual("SELECT contact.firstname, contact.lastname, account.name FROM archive.contact INNER JOIN archive.account ON contact.parentcustomerid = account.accountid", NormalizeWhitespace(converted)); } + [TestMethod] + public void Under() + { + var metadata = new AttributeMetadataCache(_service); + var fetch = @" + + + + + + + "; + + var converted = FetchXml2Sql.Convert(_service, metadata, fetch, new FetchXml2SqlOptions { ConvertFetchXmlOperatorsTo = FetchXmlOperatorConversion.SqlCalculations }, out _); + + Assert.AreEqual(NormalizeWhitespace(@" + WITH account_hierarchical(accountid) AS ( + SELECT accountid FROM account WHERE parentaccountid = 'e2218046-f778-42f6-a8a7-772d0653349b' + UNION ALL + SELECT account.accountid FROM account INNER JOIN account_hierarchical ON account.parentaccountid = account_hierarchical.accountid + ) + SELECT * FROM account WHERE accountid IN ( SELECT accountid FROM account_hierarchical )"), NormalizeWhitespace(converted)); + } + + [TestMethod] + public void EqOrUnder() + { + var metadata = new AttributeMetadataCache(_service); + var fetch = @" + + + + + + + "; + + var converted = FetchXml2Sql.Convert(_service, metadata, fetch, new FetchXml2SqlOptions { ConvertFetchXmlOperatorsTo = FetchXmlOperatorConversion.SqlCalculations }, out _); + + Assert.AreEqual(NormalizeWhitespace(@" + WITH account_hierarchical(accountid) AS ( + SELECT accountid FROM account WHERE accountid = 'e2218046-f778-42f6-a8a7-772d0653349b' + UNION ALL + SELECT account.accountid FROM account INNER JOIN account_hierarchical ON account.parentaccountid = account_hierarchical.accountid + ) + SELECT * FROM account WHERE accountid IN ( SELECT accountid FROM account_hierarchical )"), NormalizeWhitespace(converted)); + } + + [TestMethod] + public void NotUnder() + { + var metadata = new AttributeMetadataCache(_service); + var fetch = @" + + + + + + + "; + + var converted = FetchXml2Sql.Convert(_service, metadata, fetch, new FetchXml2SqlOptions { ConvertFetchXmlOperatorsTo = FetchXmlOperatorConversion.SqlCalculations }, out _); + + Assert.AreEqual(NormalizeWhitespace(@" + WITH account_hierarchical(accountid) AS ( + SELECT accountid FROM account WHERE accountid = 'e2218046-f778-42f6-a8a7-772d0653349b' + UNION ALL + SELECT account.accountid FROM account INNER JOIN account_hierarchical ON account.parentaccountid = account_hierarchical.accountid + ) + SELECT * FROM account WHERE (accountid IS NULL OR accountid NOT IN ( SELECT accountid FROM account_hierarchical ))"), NormalizeWhitespace(converted)); + } + + [TestMethod] + public void Above() + { + var metadata = new AttributeMetadataCache(_service); + var fetch = @" + + + + + + + "; + + var converted = FetchXml2Sql.Convert(_service, metadata, fetch, new FetchXml2SqlOptions { ConvertFetchXmlOperatorsTo = FetchXmlOperatorConversion.SqlCalculations }, out _); + + Assert.AreEqual(NormalizeWhitespace(@" + WITH account_hierarchical(accountid, parentaccountid) AS ( + SELECT account.accountid, account.parentaccountid FROM account INNER JOIN account AS anchor ON account.accountid = anchor.parentaccountid WHERE anchor.accountid = 'e2218046-f778-42f6-a8a7-772d0653349b' + UNION ALL + SELECT account.accountid, account.parentaccountid FROM account INNER JOIN account_hierarchical ON account.accountid = account_hierarchical.parentaccountid + ) + SELECT * FROM account WHERE accountid IN ( SELECT accountid FROM account_hierarchical )"), NormalizeWhitespace(converted)); + } + + [TestMethod] + public void EqOrAbove() + { + var metadata = new AttributeMetadataCache(_service); + var fetch = @" + + + + + + + "; + + var converted = FetchXml2Sql.Convert(_service, metadata, fetch, new FetchXml2SqlOptions { ConvertFetchXmlOperatorsTo = FetchXmlOperatorConversion.SqlCalculations }, out _); + + Assert.AreEqual(NormalizeWhitespace(@" + WITH account_hierarchical(accountid, parentaccountid) AS ( + SELECT accountid, parentaccountid FROM account WHERE accountid = 'e2218046-f778-42f6-a8a7-772d0653349b' + UNION ALL + SELECT account.accountid, account.parentaccountid FROM account INNER JOIN account_hierarchical ON account.accountid = account_hierarchical.parentaccountid + ) + SELECT * FROM account WHERE accountid IN ( SELECT accountid FROM account_hierarchical )"), NormalizeWhitespace(converted)); + } + private static string NormalizeWhitespace(string s) { - return Regex.Replace(s, "\\s+", " "); + return Regex.Replace(s, "\\s+", " ").Trim(); } } } diff --git a/MarkMpn.Sql4Cds.Engine.FetchXml.Tests/Metadata/Account.cs b/MarkMpn.Sql4Cds.Engine.FetchXml.Tests/Metadata/Account.cs index 0a9856ec..67b245ff 100644 --- a/MarkMpn.Sql4Cds.Engine.FetchXml.Tests/Metadata/Account.cs +++ b/MarkMpn.Sql4Cds.Engine.FetchXml.Tests/Metadata/Account.cs @@ -29,5 +29,12 @@ class Account [AttributeLogicalName("primarycontactid")] [RelationshipSchemaName("account_primarycontact")] public Contact PrimaryContact { get; set; } + + [AttributeLogicalName("parentaccountid")] + public EntityReference ParentAccountId { get; set; } + + [AttributeLogicalName("parentaccountid")] + [RelationshipSchemaName("account_parentaccount")] + public Account ParentAccount { get; set; } } } diff --git a/MarkMpn.Sql4Cds.Engine/FetchXml2Sql.cs b/MarkMpn.Sql4Cds.Engine/FetchXml2Sql.cs index 73be4b6b..66777927 100644 --- a/MarkMpn.Sql4Cds.Engine/FetchXml2Sql.cs +++ b/MarkMpn.Sql4Cds.Engine/FetchXml2Sql.cs @@ -1674,7 +1674,7 @@ private static BooleanExpression GetCondition(IOrganizationService org, IAttribu case @operator.eqorunder: case @operator.notunder: { - var cte = GetUnderCte(meta, new Guid(condition.value), ctes); + var cte = GetUnderCte(meta, new Guid(condition.value), ctes, condition.@operator == @operator.under); var inPred = new InPredicate { Expression = field, @@ -1692,7 +1692,6 @@ private static BooleanExpression GetCondition(IOrganizationService org, IAttribu { Identifiers = { - new Identifier { Value = cte.ExpressionName.Value }, new Identifier { Value = meta.PrimaryIdAttribute } } } @@ -1714,25 +1713,6 @@ private static BooleanExpression GetCondition(IOrganizationService org, IAttribu } } } - }, - WhereClause = condition.@operator != @operator.under ? null : new WhereClause - { - SearchCondition = new BooleanComparisonExpression - { - FirstExpression = new ColumnReferenceExpression - { - MultiPartIdentifier = new MultiPartIdentifier - { - Identifiers = - { - new Identifier { Value = cte.ExpressionName.Value }, - new Identifier { Value = "Level" } - } - } - }, - ComparisonType = BooleanComparisonType.NotEqualToBrackets, - SecondExpression = new IntegerLiteral { Value = "0" } - } } } } @@ -1756,7 +1736,7 @@ private static BooleanExpression GetCondition(IOrganizationService org, IAttribu case @operator.above: case @operator.eqorabove: { - var cte = GetAboveCte(meta, new Guid(condition.value), ctes); + var cte = GetAboveCte(meta, new Guid(condition.value), ctes, condition.@operator == @operator.above); var inPred = new InPredicate { Expression = field, @@ -1774,7 +1754,6 @@ private static BooleanExpression GetCondition(IOrganizationService org, IAttribu { Identifiers = { - new Identifier { Value = cte.ExpressionName.Value }, new Identifier { Value = meta.PrimaryIdAttribute } } } @@ -1796,25 +1775,6 @@ private static BooleanExpression GetCondition(IOrganizationService org, IAttribu } } } - }, - WhereClause = condition.@operator != @operator.above ? null : new WhereClause - { - SearchCondition = new BooleanComparisonExpression - { - FirstExpression = new ColumnReferenceExpression - { - MultiPartIdentifier = new MultiPartIdentifier - { - Identifiers = - { - new Identifier { Value = cte.ExpressionName.Value }, - new Identifier { Value = "Level" } - } - } - }, - ComparisonType = BooleanComparisonType.NotEqualToBrackets, - SecondExpression = new IntegerLiteral { Value = "0" } - } } } } @@ -1999,19 +1959,20 @@ private static FunctionCall DatePart(string datePart, ScalarExpression date) /// The metadata of the entity to recurse in /// The unique identifier of the starting record to recurse down from /// The details of all the CTEs already in the query + /// Indicates if the starting record should be excluded from the results /// The CTE that represents the required query /// /// Generates a CTE like: /// - /// account_hierarchical([Level], AccountId, ParentAccountId) AS + /// account_hierarchical(AccountId) AS /// ( - /// SELECT 0, accountid, parentaccountid FROM account WHERE accountid = 'guid' + /// SELECT accountid FROM account WHERE accountid = 'guid' /// UNION ALL - /// SELECT Level + 1, account.accountid, account.parentaccountid FROM account INNER JOIN account_hierarchical ON account.parentaccountid = account_hierarchical.accountid + /// SELECT account.accountid FROM account INNER JOIN account_hierarchical ON account.parentaccountid = account_hierarchical.accountid /// - private static CommonTableExpression GetUnderCte(EntityMetadata meta, Guid guid, IDictionary ctes) + private static CommonTableExpression GetUnderCte(EntityMetadata meta, Guid guid, IDictionary ctes, bool excludeAnchor) { - return GetCte(meta, guid, false, ctes); + return GetCte(meta, guid, false, ctes, excludeAnchor); } /// @@ -2020,19 +1981,20 @@ private static CommonTableExpression GetUnderCte(EntityMetadata meta, Guid guid, /// The metadata of the entity to recurse in /// The unique identifier of the starting record to recurse down from /// The details of all the CTEs already in the query + /// Indicates if the starting record should be excluded from the results /// The CTE that represents the required query /// /// Generates a CTE like: /// - /// account_hierarchical([Level], AccountId, ParentAccountId) AS + /// account_hierarchical(AccountId, ParentAccountId) AS /// ( - /// SELECT 0, accountid, parentaccountid FROM account WHERE accountid = 'guid' + /// SELECT accountid, parentaccountid FROM account WHERE accountid = 'guid' /// UNION ALL - /// SELECT Level + 1, account.accountid, account.parentaccountid FROM account INNER JOIN account_hierarchical ON account.accountid = account_hierarchical.parentaccountid + /// SELECT account.accountid, account.parentaccountid FROM account INNER JOIN account_hierarchical ON account.accountid = account_hierarchical.parentaccountid /// - private static CommonTableExpression GetAboveCte(EntityMetadata meta, Guid guid, IDictionary ctes) + private static CommonTableExpression GetAboveCte(EntityMetadata meta, Guid guid, IDictionary ctes, bool excludeAnchor) { - return GetCte(meta, guid, true, ctes); + return GetCte(meta, guid, true, ctes, excludeAnchor); } /// @@ -2043,7 +2005,7 @@ private static CommonTableExpression GetAboveCte(EntityMetadata meta, Guid guid, /// Indicates if the CTE should find records above the selected record (true) or below (false) /// The details of all the CTEs already in the query /// The CTE that represents the required query - private static CommonTableExpression GetCte(EntityMetadata meta, Guid guid, bool above, IDictionary ctes) + private static CommonTableExpression GetCte(EntityMetadata meta, Guid guid, bool above, IDictionary ctes, bool excludeAnchor) { if (meta == null) throw new DisconnectedException(); @@ -2070,7 +2032,6 @@ private static CommonTableExpression GetCte(EntityMetadata meta, Guid guid, bool ExpressionName = new Identifier { Value = name }, Columns = { - new Identifier { Value = "Level" }, new Identifier { Value = meta.PrimaryIdAttribute }, new Identifier { Value = parentLookupAttribute } }, @@ -2080,10 +2041,6 @@ private static CommonTableExpression GetCte(EntityMetadata meta, Guid guid, bool { SelectElements = { - new SelectScalarExpression - { - Expression = new IntegerLiteral { Value = "0" } - }, new SelectScalarExpression { Expression = new ColumnReferenceExpression @@ -2137,7 +2094,7 @@ private static CommonTableExpression GetCte(EntityMetadata meta, Guid guid, bool { Identifiers = { - new Identifier { Value = meta.PrimaryIdAttribute } + new Identifier { Value = excludeAnchor && !above ? parentLookupAttribute : meta.PrimaryIdAttribute } } } }, @@ -2152,24 +2109,6 @@ private static CommonTableExpression GetCte(EntityMetadata meta, Guid guid, bool { SelectElements = { - new SelectScalarExpression - { - Expression = new BinaryExpression - { - FirstExpression = new ColumnReferenceExpression - { - MultiPartIdentifier = new MultiPartIdentifier - { - Identifiers = - { - new Identifier { Value = "Level" } - } - } - }, - BinaryExpressionType = BinaryExpressionType.Add, - SecondExpression = new IntegerLiteral { Value = "1" } - } - }, new SelectScalarExpression { Expression = new ColumnReferenceExpression @@ -2259,6 +2198,75 @@ private static CommonTableExpression GetCte(EntityMetadata meta, Guid guid, bool } }; + if (excludeAnchor && above) + { + // Need to include a join in the anchor query to find the records above the source record but not including that record + var query = (BinaryQueryExpression)cte.QueryExpression; + var anchorQuery = (QuerySpecification)query.FirstQueryExpression; + var filter = (BooleanComparisonExpression)anchorQuery.WhereClause.SearchCondition; + var field = (ColumnReferenceExpression)filter.FirstExpression; + + anchorQuery.FromClause.TableReferences[0] = new QualifiedJoin + { + FirstTableReference = anchorQuery.FromClause.TableReferences[0], + QualifiedJoinType = QualifiedJoinType.Inner, + SecondTableReference = new NamedTableReference + { + SchemaObject = new SchemaObjectName + { + Identifiers = + { + new Identifier { Value = meta.LogicalName } + } + }, + Alias = new Identifier { Value = "anchor" } + }, + SearchCondition = new BooleanComparisonExpression + { + FirstExpression = new ColumnReferenceExpression + { + MultiPartIdentifier = new MultiPartIdentifier + { + Identifiers = + { + new Identifier { Value = meta.LogicalName }, + new Identifier { Value = meta.PrimaryIdAttribute } + } + } + }, + ComparisonType = BooleanComparisonType.Equals, + SecondExpression = new ColumnReferenceExpression + { + MultiPartIdentifier = new MultiPartIdentifier + { + Identifiers = + { + new Identifier { Value = "anchor" }, + new Identifier { Value = parentLookupAttribute } + } + } + } + } + }; + + ((ColumnReferenceExpression)((SelectScalarExpression)anchorQuery.SelectElements[0]).Expression).MultiPartIdentifier.Identifiers.Insert(0, new Identifier { Value = meta.LogicalName }); + ((ColumnReferenceExpression)((SelectScalarExpression)anchorQuery.SelectElements[1]).Expression).MultiPartIdentifier.Identifiers.Insert(0, new Identifier { Value = meta.LogicalName }); + field.MultiPartIdentifier.Identifiers.Insert(0, new Identifier { Value = "anchor" }); + } + + if (!above) + { + // Don't need the parent attribute in the CTE for "under" queries + cte.Columns.RemoveAt(1); + + var query = (BinaryQueryExpression)cte.QueryExpression; + var anchorQuery = (QuerySpecification)query.FirstQueryExpression; + var recursiveQuery = (QuerySpecification)query.SecondQueryExpression; + + anchorQuery.SelectElements.RemoveAt(1); + recursiveQuery.SelectElements.RemoveAt(1); + } + ctes.Add(name, cte); return cte;