Merged from master

AzureDataStudioExtension/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Change Log
 
+## [v7.6.0](https://github.com/MarkMpn/Sql4Cds/releases/tag/v7.6.0) - 2023-09-30
+
+Allow updating many-to-many intersect tables
+Allow folding `NOT EXISTS` predicates to FetchXML for improved performance
+Improved sorting reliability on audit and elastic tables
+Fixed KeyNotFoundException from certain joins
+Fixed NullReferenceException when applying filters to certain joins
+Preserve additional join criteria on `metadata` schema tables
+Fixed use of `SELECT *` in subqueries
+
 ## [v7.5.2](https://github.com/MarkMpn/Sql4Cds/releases/tag/v7.5.2) - 2023-09-17
 
 Fixed ordering of hash join results

MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs

@@ -1895,30 +1895,23 @@ public void NotExistsFilterCorrelated()
             Assert.AreEqual(1, plans.Length);
 
             var select = AssertNode<SelectNode>(plans[0]);
-            var filter = AssertNode<FilterNode>(select.Source);
-            Assert.AreEqual("NOT Expr3 IS NOT NULL", filter.Filter.ToSql());
-            var join = AssertNode<MergeJoinNode>(filter.Source);
-            Assert.AreEqual(QualifiedJoinType.LeftOuter, join.JoinType);
-            Assert.IsTrue(join.SemiJoin);
-            var fetch = AssertNode<FetchXmlScan>(join.LeftSource);
+            var fetch = AssertNode<FetchXmlScan>(select.Source);
+            Assert.IsTrue(fetch.UsingCustomPaging);
             AssertFetchXml(fetch, @"
                 <fetch>
                     <entity name='account'>
                         <attribute name='accountid' />
                         <attribute name='name' />
+                        <link-entity name='contact' alias='Expr2' from='parentcustomerid' to='accountid' link-type='outer'>
+                            <attribute name='contactid' />
+                            <order attribute='contactid' />
+                        </link-entity>
+                        <filter>
+                            <condition entityname='Expr2' attribute='contactid' operator='null' />
+                        </filter>
                         <order attribute='accountid' />
                     </entity>
                 </fetch>");
-            var sort = AssertNode<SortNode>(join.RightSource);
-            Assert.AreEqual("Expr2.parentcustomerid ASC", sort.Sorts[0].ToSql());
-            var subFetch = AssertNode<FetchXmlScan>(sort.Source);
-            Assert.AreEqual("Expr2", subFetch.Alias);
-            AssertFetchXml(subFetch, @"
-                <fetch distinct='true'>
-                    <entity name='contact'>
-                        <attribute name='parentcustomerid' />
-                    </entity>
-                </fetch>");
         }
 
         [TestMethod]
@@ -2769,6 +2762,41 @@ public void FoldFilterWithInClauseWithoutPrimaryKey()
             }
         }
 
+        [TestMethod]
+        public void FoldNotInToLeftOuterJoin()
+        {
+            var planBuilder = new ExecutionPlanBuilder(_dataSources.Values, new OptionsWrapper(this) { PrimaryDataSource = "uat" });
+
+            var query = "SELECT name from account where name like 'Data8%' and createdon not in (select createdon from contact where firstname = 'Mark')";
+
+            var plans = planBuilder.Build(query, null, out _);
+
+            var select = AssertNode<SelectNode>(plans[0]);
+
+            var fetch = AssertNode<FetchXmlScan>(select.Source);
+
+            Assert.IsTrue(fetch.UsingCustomPaging);
+            AssertFetchXml(fetch, @"
+                <fetch>
+                    <entity name='account'>
+                        <attribute name='name' />
+                        <attribute name='accountid' />
+                        <link-entity name='contact' alias='Expr1' from='createdon' to='createdon' link-type='outer'>
+                            <attribute name='contactid' />
+                            <filter>
+                                <condition attribute='firstname' operator='eq' value='Mark' />
+                            </filter>
+                            <order attribute='contactid' />
+                        </link-entity>
+                        <filter>
+                            <condition attribute='name' operator='like' value='Data8%' />
+                            <condition entityname='Expr1' attribute='contactid' operator='null' />
+                        </filter>
+                        <order attribute='accountid' />
+                    </entity>
+                </fetch>");
+        }
+
         [TestMethod]
         public void FoldFilterWithInClauseOnLinkEntityWithoutPrimaryKey()
         {
@@ -6027,7 +6055,7 @@ public void HashJoinUsedForDifferentDataTypes()
         {
             var planBuilder = new ExecutionPlanBuilder(_localDataSource.Values, this);
 
-            var query = "SELECT * FROM account WHERE NOT EXISTS(SELECT * FROM contact WHERE account.name = contact.createdon)";
+            var query = "SELECT * FROM account WHERE EXISTS(SELECT * FROM contact WHERE account.name = contact.createdon)";
 
             var plans = planBuilder.Build(query, null, out _);
 
@@ -6256,7 +6284,6 @@ WHERE    [union. all].eln IN ('systemuser', 'businessunit')
          AND [union. all].logicalname IN ('createdon')
 ORDER BY [union. all].eln";
 
-
             var plans = planBuilder.Build(query, null, out _);
 
             var select = AssertNode<SelectNode>(plans[0]);
@@ -6303,5 +6330,60 @@ AND [union. all].logicalname IN ('createdon')
             Assert.AreEqual(MetadataConditionOperator.In, mq3.Query.AttributeQuery.Criteria.Conditions[0].ConditionOperator);
             CollectionAssert.AreEqual(new[] { "createdon" }, (string[])mq3.Query.AttributeQuery.Criteria.Conditions[0].Value);
         }
+
+        [TestMethod]
+        public void PreserveAdditionalFiltersInMetadataJoinConditions()
+        {
+            var planBuilder = new ExecutionPlanBuilder(_dataSources.Values, new OptionsWrapper(this) { PrimaryDataSource = "uat" });
+
+            var query = @"
+SELECT e.logicalname,
+       a.logicalname,
+       a.targets
+FROM   metadata.entity AS e
+       INNER JOIN
+       metadata.attribute AS a
+       ON e.logicalname = a.entitylogicalname
+          AND a.targets IS NOT NULL
+WHERE  e.logicalname IN ('systemuser');";
+
+            var plans = planBuilder.Build(query, null, out _);
+
+            var select = AssertNode<SelectNode>(plans[0]);
+            var filter = AssertNode<FilterNode>(select.Source);
+            Assert.AreEqual("a.targets IS NOT NULL", filter.Filter.ToSql());
+            var metadata = AssertNode<MetadataQueryNode>(filter.Source);
+            Assert.AreEqual("e", metadata.EntityAlias);
+            Assert.AreEqual("a", metadata.AttributeAlias);
+            Assert.AreEqual(MetadataSource.Entity | MetadataSource.Attribute, metadata.MetadataSource);
+            CollectionAssert.AreEquivalent(new[] { nameof(EntityMetadata.LogicalName) }, metadata.Query.Properties.PropertyNames);
+            CollectionAssert.AreEquivalent(new[] { nameof(AttributeMetadata.LogicalName), nameof(LookupAttributeMetadata.Targets) }, metadata.Query.AttributeQuery.Properties.PropertyNames);
+            Assert.AreEqual(nameof(EntityMetadata.LogicalName), metadata.Query.Criteria.Conditions[0].PropertyName);
+            Assert.AreEqual(MetadataConditionOperator.In, metadata.Query.Criteria.Conditions[0].ConditionOperator);
+            CollectionAssert.AreEquivalent(new[] { "systemuser" }, (string[])metadata.Query.Criteria.Conditions[0].Value);
+        }
+
+        [TestMethod]
+        public void FoldFilterToJoinWithAlias()
+        {
+            // https://github.com/MarkMpn/Sql4Cds/issues/364
+            // Filter is applied to a join with a LHS of FetchXmlScan, which the filter can be entirely folded to
+            // Exception is thrown when trying to fold the remaining null filter to the RHS of the join
+
+            var planBuilder = new ExecutionPlanBuilder(_dataSources.Values, new OptionsWrapper(this) { PrimaryDataSource = "uat" });
+
+            var query = @"
+SELECT app.*
+FROM   (SELECT a.accountid, c.contactid, tot = sum(1)
+FROM account a
+INNER JOIN contact c ON a.accountid = c.parentcustomerid
+WHERE a.name = 'Data8'
+GROUP BY a.accountid, c.contactid
+HAVING sum(1) > 1) AS dups
+INNER JOIN account app ON app.accountid = dups.accountid
+WHERE  app.name = 'Data8'";
+
+            var plans = planBuilder.Build(query, null, out _);
+        }
     }
 }

MarkMpn.Sql4Cds.Engine/Ado/Sql4CdsCommand.cs

@@ -256,7 +256,7 @@ protected override DbDataReader ExecuteDbDataReader(CommandBehavior behavior)
 
                     if (Parameters.Count > 0)
                     {
-                        var dom = new TSql150Parser(_connection.Options.QuotedIdentifiers);
+                        var dom = new TSql160Parser(_connection.Options.QuotedIdentifiers);
                         var fragment = dom.Parse(new StringReader(CommandText), out _);
                         var variables = new VariableCollectingVisitor();
                         fragment.Accept(variables);

MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseNode.cs

@@ -63,6 +63,13 @@ protected string GetDisplayName(int count, EntityMetadata meta)
                 meta.LogicalName;
         }
 
+        /// <summary>
+        /// Notifies the node that query folding is complete
+        /// </summary>
+        public virtual void FinishedFolding()
+        {
+        }
+
         public override string ToString()
         {
             return System.Text.RegularExpressions.Regex.Replace(GetType().Name.Replace("Node", ""), "([a-z])([A-Z])", "$1 $2");

MarkMpn.Sql4Cds.Engine/ExecutionPlan/DeleteNode.cs

@@ -212,14 +212,10 @@ private OrganizationRequest CreateDeleteRequest(EntityMetadata meta, Entity enti
 
                 if (meta.LogicalName == "listmember")
                 {
-                    return new OrganizationRequest
+                    return new RemoveMemberListRequest
                     {
-                        RequestName = "RemoveMemberList",
-                        Parameters = new ParameterCollection
-                        {
-                            ["ListId"] = id,
-                            ["EntityId"] = secondaryId
-                        }
+                        ListId = id,
+                        EntityId = secondaryId
                     };
                 }
 
@@ -228,7 +224,7 @@ private OrganizationRequest CreateDeleteRequest(EntityMetadata meta, Entity enti
                 return new DisassociateRequest
                 {
                     Target = new EntityReference(relationship.Entity1LogicalName, id),
-                    RelatedEntities = new EntityReferenceCollection(new[] { new EntityReference(relationship.Entity2LogicalName, secondaryId) }),
+                    RelatedEntities = new EntityReferenceCollection { new EntityReference(relationship.Entity2LogicalName, secondaryId) },
                     Relationship = new Relationship(relationship.SchemaName) { PrimaryEntityRole = EntityRole.Referencing }
                 };
             }

MarkMpn.Sql4Cds.Engine/ExecutionPlan/ExpressionExtensions.cs

@@ -26,6 +26,8 @@ namespace MarkMpn.Sql4Cds.Engine.ExecutionPlan
     /// </summary>
     static class ExpressionExtensions
     {
+        private static TSqlParser _parser = new TSql160Parser(false);
+
         /// <summary>
         /// Gets the type of value that will be generated by an expression
         /// </summary>
@@ -1819,19 +1821,7 @@ private static bool IsValidIdentifier(string identifier)
             if (String.IsNullOrEmpty(identifier))
                 return false;
 
-            // The first character must be a letter, _, @ or #
-            // However, @ or # are special and indicate variables or temporary tables, so don't allow them as
-            // this is designed for columns only
-            if (!Char.IsLetter(identifier[0]) && identifier[0] != '_')
-                return false;
-
-            // Subsequent characters can be letters, numbers, @, $, # or _
-            if (!identifier.All(ch => Char.IsLetterOrDigit(ch) || ch == '_' || ch == '@' || ch == '$' || ch == '#'))
-                return false;
-
-            // TODO: The identifier must not be a reserved word
-
-            return true;
+            return _parser.ValidateIdentifier(identifier);
         }
 
         /// <summary>

MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs

@@ -235,12 +235,6 @@ public bool RequiresCustomPaging(IDictionary<string, DataSource> dataSources)
             return false;
         }
 
-        public override IEnumerable<Entity> Execute(NodeExecutionContext context)
-        {
-            ReturnFullSchema = false;
-            return base.Execute(context);
-        }
-
         protected override IEnumerable<Entity> ExecuteInternal(NodeExecutionContext context)
         {
             PagesRetrieved = 0;
@@ -1643,7 +1637,7 @@ public override void AddRequiredColumns(NodeCompilationContext context, IList<st
 
                 var attr = AddAttribute(normalizedCol, null, dataSource.Metadata, out _, out var linkEntity);
 
-                if (mapping != null)
+                if (mapping != null && attr != null)
                 {
                     if (attr.name != parts[1] && IsValidAlias(parts[1]))
                         attr.alias = parts[1];
@@ -1787,6 +1781,11 @@ private void SetDefaultPageSize(NodeCompilationContext context)
             ReturnFullSchema = fullSchema;
         }
 
+        public override void FinishedFolding()
+        {
+            ReturnFullSchema = false;
+        }
+
         protected override RowCountEstimate EstimateRowsOutInternal(NodeCompilationContext context)
         {
             if (FetchXml.aggregateSpecified && FetchXml.aggregate)