Keep column ordering consistent

MarkMpn · Apr 3, 2023 · 4b10345 · 4b10345
1 parent 5e8ecd5
commit 4b10345
Show file tree

Hide file tree

Showing 16 changed files with 151 additions and 41 deletions.
diff --git a/MarkMpn.Sql4Cds.Engine.Tests/AdoProviderTests.cs b/MarkMpn.Sql4Cds.Engine.Tests/AdoProviderTests.cs
@@ -1080,6 +1080,8 @@ public void MergeSemiJoin()
             using (var con = new Sql4CdsConnection(_dataSources))
             using (var cmd = con.CreateCommand())
             {
+                cmd.CommandTimeout = 0;
+
                 cmd.CommandText = "insert into account (name) values ('data8')";
                 cmd.ExecuteNonQuery();
                 cmd.ExecuteNonQuery();

diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/AliasNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/AliasNode.cs
@@ -155,7 +155,7 @@ public override INodeSchema GetSchema(NodeCompilationContext context)
         {
             // Map the base names to the alias names
             var sourceSchema = Source.GetSchema(context);
-            var schema = new Dictionary<string, DataTypeReference>(StringComparer.OrdinalIgnoreCase);
+            var schema = new ColumnList();
             var aliases = new Dictionary<string, IReadOnlyList<string>>();
             var primaryKey = (string)null;
             var mappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
@@ -218,7 +218,7 @@ public override INodeSchema GetSchema(NodeCompilationContext context)
                 sortOrder: sortOrder);
         }
 
-        private void AddSchemaColumn(string outputColumn, string sourceColumn, Dictionary<string, DataTypeReference> schema, Dictionary<string, IReadOnlyList<string>> aliases, ref string primaryKey, Dictionary<string, string> mappings, INodeSchema sourceSchema)
+        private void AddSchemaColumn(string outputColumn, string sourceColumn, ColumnList schema, Dictionary<string, IReadOnlyList<string>> aliases, ref string primaryKey, Dictionary<string, string> mappings, INodeSchema sourceSchema)
         {
             if (!sourceSchema.ContainsColumn(sourceColumn, out var normalized))
                 return;

diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseAggregateNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseAggregateNode.cs
@@ -178,7 +178,7 @@ public override INodeSchema GetSchema(NodeCompilationContext context)
         {
             var sourceSchema = Source.GetSchema(context);
             var expressionContext = new ExpressionCompilationContext(context, sourceSchema, null);
-            var schema = new Dictionary<string, DataTypeReference>(StringComparer.OrdinalIgnoreCase);
+            var schema = new ColumnList();
             var aliases = new Dictionary<string, IReadOnlyList<string>>(StringComparer.OrdinalIgnoreCase);
             var primaryKey = (string)null;
             var notNullColumns = new List<string>();

diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDmlNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDmlNode.cs
@@ -211,7 +211,7 @@ protected List<Entity> GetDmlSourceEntities(NodeExecutionContext context, out IN
                 // Store the values under the column index as well as name for compatibility with INSERT ... SELECT ...
                 var dataTable = new DataTable();
                 var schemaTable = dataReader.GetSchemaTable();
-                var columnTypes = new Dictionary<string, DataTypeReference>(StringComparer.OrdinalIgnoreCase);
+                var columnTypes = new ColumnList();
                 var targetDataSource = context.DataSources[DataSource];
 
                 for (var i = 0; i < schemaTable.Rows.Count; i++)

diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseJoinNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseJoinNode.cs
@@ -17,6 +17,7 @@ abstract class BaseJoinNode : BaseDataNode
         private INodeSchema _lastLeftSchema;
         private INodeSchema _lastRightSchema;
         private INodeSchema _lastSchema;
+        private bool _lastSchemaIncludedSemiJoin;
 
         /// <summary>
         /// The first data source to merge
@@ -115,10 +116,10 @@ protected virtual INodeSchema GetSchema(NodeCompilationContext context, bool inc
             var outerSchema = LeftSource.GetSchema(context);
             var innerSchema = GetRightSchema(context);
 
-            if (outerSchema == _lastLeftSchema && innerSchema == _lastRightSchema)
+            if (outerSchema == _lastLeftSchema && innerSchema == _lastRightSchema && includeSemiJoin == _lastSchemaIncludedSemiJoin)
                 return _lastSchema;
 
-            var schema = new Dictionary<string, DataTypeReference>(StringComparer.OrdinalIgnoreCase);
+            var schema = new ColumnList();
             var aliases = new Dictionary<string, IReadOnlyList<string>>(StringComparer.OrdinalIgnoreCase);
             var primaryKey = GetPrimaryKey(outerSchema, innerSchema);
             var notNullColumns = new List<string>();
@@ -162,6 +163,7 @@ protected virtual INodeSchema GetSchema(NodeCompilationContext context, bool inc
                 aliases: aliases,
                 notNullColumns: notNullColumns,
                 sortOrder: GetSortOrder(outerSchema, innerSchema));
+            _lastSchemaIncludedSemiJoin = includeSemiJoin;
             return _lastSchema;
         }
 

diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/ComputeScalarNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/ComputeScalarNode.cs
@@ -54,7 +54,7 @@ public override INodeSchema GetSchema(NodeCompilationContext context)
             // Copy the source schema and add in the additional computed columns
             var sourceSchema = Source.GetSchema(context);
             var expressionCompilationContext = new ExpressionCompilationContext(context, sourceSchema, null);
-            var schema = new Dictionary<string, DataTypeReference>(sourceSchema.Schema.Count, StringComparer.OrdinalIgnoreCase);
+            var schema = new ColumnList();
 
             foreach (var col in sourceSchema.Schema)
                 schema[col.Key] = col.Value;

diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/ConcatenateNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/ConcatenateNode.cs
@@ -48,7 +48,7 @@ protected override IEnumerable<Entity> ExecuteInternal(NodeExecutionContext cont
 
         public override INodeSchema GetSchema(NodeCompilationContext context)
         {
-            var schema = new Dictionary<string, DataTypeReference>(StringComparer.OrdinalIgnoreCase);
+            var schema = new ColumnList();
 
             var sourceSchema = Sources[0].GetSchema(context);
 

diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/ConstantScanNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/ConstantScanNode.cs
@@ -31,7 +31,7 @@ class ConstantScanNode : BaseDataNode
         /// The types of values to be returned
         /// </summary>
         [Browsable(false)]
-        public Dictionary<string, DataTypeReference> Schema { get; private set; } = new Dictionary<string, DataTypeReference>();
+        public IDictionary<string, DataTypeReference> Schema { get; private set; } = new ColumnList();
 
         protected override IEnumerable<Entity> ExecuteInternal(NodeExecutionContext context)
         {
@@ -56,9 +56,14 @@ public override IEnumerable<IExecutionPlanNode> GetSources()
 
         public override INodeSchema GetSchema(NodeCompilationContext context)
         {
+            var schema = new ColumnList();
+
+            foreach (var col in Schema)
+                schema[PrefixWithAlias(col.Key)] = col.Value;
+
             return new NodeSchema(
                 primaryKey: null,
-                schema: Schema.ToDictionary(kvp => PrefixWithAlias(kvp.Key), kvp => kvp.Value, StringComparer.OrdinalIgnoreCase),
+                schema: schema,
                 aliases: Schema.ToDictionary(kvp => kvp.Key, kvp => (IReadOnlyList<string>) new List<string> { PrefixWithAlias(kvp.Key) }, StringComparer.OrdinalIgnoreCase),
                 notNullColumns: null,
                 sortOrder: null);

diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/ExecuteMessageNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/ExecuteMessageNode.cs
@@ -92,7 +92,7 @@ class ExecuteMessageNode : BaseDataNode, IDmlQueryExecutionPlanNode
         /// The types of values to be returned
         /// </summary>
         [Browsable(false)]
-        public Dictionary<string, DataTypeReference> Schema { get; private set; } = new Dictionary<string, DataTypeReference>();
+        public IDictionary<string, DataTypeReference> Schema { get; private set; } = new ColumnList();
 
         /// <summary>
         /// Indicates if custom plugins should be skipped
@@ -148,9 +148,14 @@ private bool GetBypassPluginExecution(IList<OptimizerHint> queryHints, IQueryExe
 
         public override INodeSchema GetSchema(NodeCompilationContext context)
         {
+            var schema = new ColumnList();
+
+            foreach (var col in Schema)
+                schema[PrefixWithAlias(col.Key)] = col.Value;
+
             return new NodeSchema(
                 primaryKey: null,
-                schema: Schema.ToDictionary(kvp => PrefixWithAlias(kvp.Key), kvp => kvp.Value, StringComparer.OrdinalIgnoreCase),
+                schema: schema,
                 aliases: Schema.ToDictionary(kvp => kvp.Key, kvp => (IReadOnlyList<string>)new List<string> { PrefixWithAlias(kvp.Key) }, StringComparer.OrdinalIgnoreCase),
                 notNullColumns: null,
                 sortOrder: null);

diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs
@@ -655,7 +655,7 @@ public override INodeSchema GetSchema(NodeCompilationContext context)
             var entity = FetchXml.Items.OfType<FetchEntityType>().Single();
             var meta = dataSource.Metadata[entity.name];
 
-            var schema = new Dictionary<string, DataTypeReference>(StringComparer.OrdinalIgnoreCase);
+            var schema = new ColumnList();
             var aliases = new Dictionary<string, IReadOnlyList<string>>(StringComparer.OrdinalIgnoreCase);
             var primaryKey = FetchXml.aggregate ? null : $"{Alias}.{meta.PrimaryIdAttribute}";
             var notNullColumns = new HashSet<string>();
@@ -759,7 +759,7 @@ internal static bool IsValidAlias(string alias)
             return Regex.IsMatch(alias, "^[A-Za-z_][A-Za-z0-9_]*$");
         }
 
-        private void AddSchemaAttributes(DataSource dataSource, Dictionary<string, DataTypeReference> schema, Dictionary<string, IReadOnlyList<string>> aliases, ref string primaryKey, HashSet<string> notNullColumns, List<string> sortOrder, string entityName, string alias, object[] items, bool innerJoin, bool requireTablePrefix)
+        private void AddSchemaAttributes(DataSource dataSource, ColumnList schema, Dictionary<string, IReadOnlyList<string>> aliases, ref string primaryKey, HashSet<string> notNullColumns, List<string> sortOrder, string entityName, string alias, object[] items, bool innerJoin, bool requireTablePrefix)
         {
             if (items == null && !ReturnFullSchema)
                 return;
@@ -768,7 +768,7 @@ private void AddSchemaAttributes(DataSource dataSource, Dictionary<string, DataT
 
             if (ReturnFullSchema && !FetchXml.aggregate)
             {
-                foreach (var attrMetadata in meta.Attributes)
+                foreach (var attrMetadata in meta.Attributes.OrderBy(a => a.LogicalName))
                 {
                     if (attrMetadata.IsValidForRead == false)
                         continue;
@@ -918,7 +918,7 @@ private void AddSchemaAttributes(DataSource dataSource, Dictionary<string, DataT
             }
         }
 
-        private void AddNotNullFilters(Dictionary<string, DataTypeReference> schema, Dictionary<string, IReadOnlyList<string>> aliases, HashSet<string> notNullColumns, string alias, filter filter)
+        private void AddNotNullFilters(ColumnList schema, Dictionary<string, IReadOnlyList<string>> aliases, HashSet<string> notNullColumns, string alias, filter filter)
         {
             if (filter.Items == null)
                 return;
@@ -941,7 +941,7 @@ private void AddNotNullFilters(Dictionary<string, DataTypeReference> schema, Dic
                 AddNotNullFilters(schema, aliases, notNullColumns, alias, subFilter);
         }
 
-        private void AddSchemaAttribute(DataSource dataSource, Dictionary<string, DataTypeReference> schema, Dictionary<string, IReadOnlyList<string>> aliases, HashSet<string> notNullColumns, string fullName, string simpleName, DataTypeReference type, AttributeMetadata attrMetadata, bool innerJoin)
+        private void AddSchemaAttribute(DataSource dataSource, ColumnList schema, Dictionary<string, IReadOnlyList<string>> aliases, HashSet<string> notNullColumns, string fullName, string simpleName, DataTypeReference type, AttributeMetadata attrMetadata, bool innerJoin)
         {
             var notNull = innerJoin && (attrMetadata.RequiredLevel?.Value == AttributeRequiredLevel.SystemRequired || attrMetadata.LogicalName == "createdon" || attrMetadata.LogicalName == "createdby" || attrMetadata.AttributeOf == "createdby");
 
@@ -969,7 +969,7 @@ private void AddSchemaAttribute(DataSource dataSource, Dictionary<string, DataTy
             }
         }
 
-        private void AddSchemaAttribute(Dictionary<string, DataTypeReference> schema, Dictionary<string, IReadOnlyList<string>> aliases, HashSet<string> notNullColumns, string fullName, string simpleName, DataTypeReference type, bool notNull)
+        private void AddSchemaAttribute(ColumnList schema, Dictionary<string, IReadOnlyList<string>> aliases, HashSet<string> notNullColumns, string fullName, string simpleName, DataTypeReference type, bool notNull)
         {
             schema[fullName] = type;
 

diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/GlobalOptionSetQueryNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/GlobalOptionSetQueryNode.cs
@@ -117,7 +117,7 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext
 
         public override INodeSchema GetSchema(NodeCompilationContext context)
         {
-            var schema = new Dictionary<string, DataTypeReference>(StringComparer.OrdinalIgnoreCase);
+            var schema = new ColumnList();
             var aliases = new Dictionary<string, IReadOnlyList<string>>(StringComparer.OrdinalIgnoreCase);
 
             foreach (var prop in _optionsetProps.Values)

diff --git a/MarkMpn.Sql4Cds.Engine/ExecutionPlan/MetadataQueryNode.cs b/MarkMpn.Sql4Cds.Engine/ExecutionPlan/MetadataQueryNode.cs
@@ -441,7 +441,7 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext
 
         public override INodeSchema GetSchema(NodeCompilationContext context)
         {
-            var schema = new Dictionary<string, DataTypeReference>(StringComparer.OrdinalIgnoreCase);
+            var schema = new ColumnList();
             var aliases = new Dictionary<string, IReadOnlyList<string>>();
             var primaryKey = (string)null;
             var notNullColumns = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
@@ -456,7 +456,7 @@ public override INodeSchema GetSchema(NodeCompilationContext context)
                 if (Query.Properties != null)
                     entityProps = entityProps.Where(p => Query.Properties.AllProperties || Query.Properties.PropertyNames.Contains(p.PropertyName, StringComparer.OrdinalIgnoreCase));
 
-                foreach (var prop in entityProps)
+                foreach (var prop in entityProps.OrderBy(p => p.SqlName))
                 {
                     var fullName = $"{EntityAlias}.{prop.SqlName}";
                     schema[fullName] = prop.SqlType;
@@ -483,7 +483,7 @@ public override INodeSchema GetSchema(NodeCompilationContext context)
                 if (Query.AttributeQuery?.Properties != null)
                     attributeProps = attributeProps.Where(p => Query.AttributeQuery.Properties.AllProperties || Query.AttributeQuery.Properties.PropertyNames.Contains(p.PropertyName, StringComparer.OrdinalIgnoreCase));
 
-                foreach (var prop in attributeProps)
+                foreach (var prop in attributeProps.OrderBy(p => p.SqlName))
                 {
                     var fullName = $"{AttributeAlias}.{prop.SqlName}";
                     schema[fullName] = prop.SqlType;
@@ -511,7 +511,7 @@ public override INodeSchema GetSchema(NodeCompilationContext context)
                 if (Query.RelationshipQuery?.Properties != null)
                     relationshipProps = relationshipProps.Where(p => Query.RelationshipQuery.Properties.AllProperties || Query.RelationshipQuery.Properties.PropertyNames.Contains(p.PropertyName, StringComparer.OrdinalIgnoreCase));
 
-                foreach (var prop in relationshipProps)
+                foreach (var prop in relationshipProps.OrderBy(p => p.SqlName))
                 {
                     var fullName = $"{OneToManyRelationshipAlias}.{prop.SqlName}";
                     schema[fullName] = prop.SqlType;
@@ -539,7 +539,7 @@ public override INodeSchema GetSchema(NodeCompilationContext context)
                 if (Query.RelationshipQuery?.Properties != null)
                     relationshipProps = relationshipProps.Where(p => Query.RelationshipQuery.Properties.AllProperties || Query.RelationshipQuery.Properties.PropertyNames.Contains(p.PropertyName, StringComparer.OrdinalIgnoreCase));
 
-                foreach (var prop in relationshipProps)
+                foreach (var prop in relationshipProps.OrderBy(p => p.SqlName))
                 {
                     var fullName = $"{ManyToOneRelationshipAlias}.{prop.SqlName}";
                     schema[fullName] = prop.SqlType;
@@ -567,7 +567,7 @@ public override INodeSchema GetSchema(NodeCompilationContext context)
                 if (Query.RelationshipQuery?.Properties != null)
                     relationshipProps = relationshipProps.Where(p => Query.RelationshipQuery.Properties.AllProperties || Query.RelationshipQuery.Properties.PropertyNames.Contains(p.PropertyName, StringComparer.OrdinalIgnoreCase));
 
-                foreach (var prop in relationshipProps)
+                foreach (var prop in relationshipProps.OrderBy(p => p.SqlName))
                 {
                     var fullName = $"{ManyToManyRelationshipAlias}.{prop.SqlName}";
                     schema[fullName] = prop.SqlType;