Initial basic CTE implementation

MarkMpn.Sql4Cds.Engine.Tests/CteTests.cs

@@ -0,0 +1,135 @@
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Data.SqlTypes;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Runtime.Remoting.Contexts;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Web.Services.Description;
+using System.Xml.Serialization;
+using FakeXrmEasy;
+using FakeXrmEasy.Extensions;
+using MarkMpn.Sql4Cds.Engine.ExecutionPlan;
+using Microsoft.SqlServer.TransactSql.ScriptDom;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Microsoft.Xrm.Sdk;
+using Microsoft.Xrm.Sdk.Metadata;
+using Microsoft.Xrm.Sdk.Metadata.Query;
+using Microsoft.Xrm.Sdk.Query;
+using Microsoft.Xrm.Tooling.Connector;
+
+namespace MarkMpn.Sql4Cds.Engine.Tests
+{
+    [TestClass]
+    public class CteTests : FakeXrmEasyTestsBase, IQueryExecutionOptions
+    {
+        private List<JoinOperator> _supportedJoins = new List<JoinOperator>
+        {
+            JoinOperator.Inner,
+            JoinOperator.LeftOuter
+        };
+
+        CancellationToken IQueryExecutionOptions.CancellationToken => CancellationToken.None;
+
+        bool IQueryExecutionOptions.BlockUpdateWithoutWhere => false;
+
+        bool IQueryExecutionOptions.BlockDeleteWithoutWhere => false;
+
+        bool IQueryExecutionOptions.UseBulkDelete => false;
+
+        int IQueryExecutionOptions.BatchSize => 1;
+
+        bool IQueryExecutionOptions.UseTDSEndpoint => false;
+
+        int IQueryExecutionOptions.MaxDegreeOfParallelism => 10;
+
+        bool IQueryExecutionOptions.ColumnComparisonAvailable => true;
+
+        bool IQueryExecutionOptions.UseLocalTimeZone => true;
+
+        List<JoinOperator> IQueryExecutionOptions.JoinOperatorsAvailable => _supportedJoins;
+
+        bool IQueryExecutionOptions.BypassCustomPlugins => false;
+
+        void IQueryExecutionOptions.ConfirmInsert(ConfirmDmlStatementEventArgs e)
+        {
+        }
+
+        void IQueryExecutionOptions.ConfirmDelete(ConfirmDmlStatementEventArgs e)
+        {
+        }
+
+        void IQueryExecutionOptions.ConfirmUpdate(ConfirmDmlStatementEventArgs e)
+        {
+        }
+
+        bool IQueryExecutionOptions.ContinueRetrieve(int count)
+        {
+            return true;
+        }
+
+        void IQueryExecutionOptions.Progress(double? progress, string message)
+        {
+        }
+
+        string IQueryExecutionOptions.PrimaryDataSource => "local";
+
+        Guid IQueryExecutionOptions.UserId => Guid.NewGuid();
+
+        bool IQueryExecutionOptions.QuotedIdentifiers => true;
+
+        public ColumnOrdering ColumnOrdering => ColumnOrdering.Alphabetical;
+
+        [TestMethod]
+        public void SimpleSelect()
+        {
+            var planBuilder = new ExecutionPlanBuilder(_localDataSource.Values, this);
+
+            var query = @"
+                WITH cte AS (SELECT accountid, name FROM account)
+                SELECT * FROM cte";
+
+            var plans = planBuilder.Build(query, null, out _);
+
+            Assert.AreEqual(1, plans.Length);
+
+            var select = AssertNode<SelectNode>(plans[0]);
+            var fetch = AssertNode<FetchXmlScan>(select.Source);
+            AssertFetchXml(fetch, @"
+                <fetch>
+                    <entity name='account'>
+                        <attribute name='accountid' />
+                        <attribute name='name' />
+                    </entity>
+                </fetch>");
+        }
+
+        private T AssertNode<T>(IExecutionPlanNode node) where T : IExecutionPlanNode
+        {
+            Assert.IsInstanceOfType(node, typeof(T));
+            return (T)node;
+        }
+
+        private void AssertFetchXml(FetchXmlScan node, string fetchXml)
+        {
+            try
+            {
+                var serializer = new XmlSerializer(typeof(FetchXml.FetchType));
+                using (var reader = new StringReader(fetchXml))
+                {
+                    var fetch = (FetchXml.FetchType)serializer.Deserialize(reader);
+                    PropertyEqualityAssert.Equals(fetch, node.FetchXml);
+                }
+            }
+            catch (AssertFailedException ex)
+            {
+                Assert.Fail($"Expected:\r\n{fetchXml}\r\n\r\nActual:\r\n{node.FetchXmlString}\r\n\r\n{ex.Message}");
+            }
+        }
+    }
+}

MarkMpn.Sql4Cds.Engine.Tests/MarkMpn.Sql4Cds.Engine.Tests.csproj

@@ -78,6 +78,7 @@
   <ItemGroup>
     <Compile Include="CollationTests.cs" />
     <Compile Include="ExecutionPlanNodeTests.cs" />
+    <Compile Include="CteTests.cs" />
     <Compile Include="ExecutionPlanTests.cs" />
     <Compile Include="ExpressionTests.cs" />
     <Compile Include="FakeXrmEasyTestsBase.cs" />

MarkMpn.Sql4Cds.Engine/ExecutionPlan/SelectNode.cs

@@ -117,11 +117,12 @@ internal static void FoldFetchXmlColumns(IDataExecutionPlanNode source, List<Sel
                         if (col.SourceColumn == null)
                         {
                             // Add an all-attributes to the main entity and all link-entities
-                            fetchXml.Entity.AddItem(new allattributes());
+                            if (!fetchXml.HiddenAliases.Contains(fetchXml.Alias))
+                                fetchXml.Entity.AddItem(new allattributes());
 
                             foreach (var link in fetchXml.Entity.GetLinkEntities())
                             {
-                                if (link.SemiJoin)
+                                if (link.SemiJoin || fetchXml.HiddenAliases.Contains(link.alias))
                                     continue;
 
                                 link.AddItem(new allattributes());

MarkMpn.Sql4Cds.Engine/ExecutionPlanBuilder.cs

@@ -20,6 +20,7 @@ class ExecutionPlanBuilder
     {
         private ExpressionCompilationContext _staticContext;
         private NodeCompilationContext _nodeContext;
+        private Dictionary<string, IDataExecutionPlanNodeInternal> _cteSubplans;
 
         public ExecutionPlanBuilder(IEnumerable<DataSource> dataSources, IQueryExecutionOptions options)
         {
@@ -166,7 +167,38 @@ private void ConvertStatement(TSqlStatement statement, ExecutionPlanOptimizer op
             var originalSql = statement.ToSql();
 
             IRootExecutionPlanNodeInternal[] plans;
-            var hints = statement is StatementWithCtesAndXmlNamespaces stmtWithCtes ? stmtWithCtes.OptimizerHints : null;
+            IList<OptimizerHint> hints = null;
+            _cteSubplans = new Dictionary<string, IDataExecutionPlanNodeInternal>(StringComparer.OrdinalIgnoreCase);
+
+            if (statement is StatementWithCtesAndXmlNamespaces stmtWithCtes)
+            {
+                hints = stmtWithCtes.OptimizerHints;
+
+                if (stmtWithCtes.WithCtesAndXmlNamespaces != null)
+                {
+                    foreach (var cte in stmtWithCtes.WithCtesAndXmlNamespaces.CommonTableExpressions)
+                    {
+                        if (_cteSubplans.ContainsKey(cte.ExpressionName.Value))
+                            throw new NotSupportedQueryFragmentException($"A CTE with the name '{cte.ExpressionName.Value}' has already been declared.", cte.ExpressionName);
+
+                        // If the CTE isn't recursive then we can just convert it to a subquery
+                        var plan = ConvertSelectStatement(cte.QueryExpression, hints, null, null, _nodeContext);
+
+                        // Apply column aliases
+                        if (cte.Columns.Count > 0)
+                        {
+                            // TODO: What if a different number of columns?
+
+                            plan.ExpandWildcardColumns(_nodeContext);
+
+                            for (var i = 0; i < cte.Columns.Count; i++)
+                                plan.ColumnSet[i].OutputColumn = cte.Columns[i].Value;
+                        }
+
+                        _cteSubplans.Add(cte.ExpressionName.Value, new AliasNode(plan, cte.ExpressionName, _nodeContext));
+                    }
+                }
+            }
 
             if (statement is SelectStatement select)
                 plans = new[] { ConvertSelectStatement(select) };
@@ -1658,9 +1690,6 @@ private IRootExecutionPlanNodeInternal ConvertSelectStatement(SelectStatement se
             if (select.On != null)
                 throw new NotSupportedQueryFragmentException("Unsupported ON clause", select.On);
 
-            if (select.WithCtesAndXmlNamespaces != null)
-                throw new NotSupportedQueryFragmentException("Unsupported CTE clause", select.WithCtesAndXmlNamespaces);
-
             var variableAssignments = new List<string>();
             SelectElement firstNonSetSelectElement = null;
 
@@ -3471,6 +3500,9 @@ private IDataExecutionPlanNodeInternal ConvertTableReference(TableReference refe
         {
             if (reference is NamedTableReference table)
             {
+                if (table.SchemaObject.Identifiers.Count == 1 && _cteSubplans.TryGetValue(table.SchemaObject.BaseIdentifier.Value, out var cteSubplan))
+                    return cteSubplan;
+
                 var dataSource = SelectDataSource(table.SchemaObject);
                 var entityName = table.SchemaObject.BaseIdentifier.Value;
 

MarkMpn.Sql4Cds.Tests/AutocompleteTests.cs

@@ -390,5 +390,19 @@ DECLARE @x varchar
 
             CollectionAssert.IsSubsetOf(new[] { "@i", "@x", "@@ROWCOUNT", "@@IDENTITY", "@@SERVERNAME", "@@VERSION" }, suggestions);
         }
+
+        [TestMethod]
+        public void SuggestVariablesWithoutFrom()
+        {
+            var sql = @"
+                DECLARE @i int
+                DECLARE @x varchar
+
+                SELECT ";
+
+            var suggestions = _autocomplete.GetSuggestions(sql, sql.Length - 1).Select(s => s.Text).ToList();
+
+            CollectionAssert.IsSubsetOf(new[] { "@i", "@x", "@@ROWCOUNT", "@@IDENTITY", "@@SERVERNAME", "@@VERSION" }, suggestions);
+        }
     }
 }