Merge pull request #490 from MarkMpn/v9-fixes

V9 fixes

MarkMpn.Sql4Cds.Engine.Tests/FakeXrmDataSource.cs

@@ -39,6 +39,8 @@ public FakeXrmDataSource()
 
         public override bool ColumnComparisonAvailable => _columnComparisonAvailable;
 
+        public override bool CrossTableColumnComparisonAvailable => _columnComparisonAvailable;
+
         public override bool OrderByEntityNameAvailable => _orderByEntityNameAvailable;
 
         public override List<JoinOperator> JoinOperatorsAvailable => _joinOperators;

MarkMpn.Sql4Cds.Engine/AttributeMetadataCache.cs

@@ -256,6 +256,12 @@ protected void OnMetadataLoading(MetadataLoadingEventArgs args)
         {
             MetadataLoading?.Invoke(this, args);
         }
+
+        /// <inheritdoc/>
+        public IEnumerable<EntityMetadata> GetAllEntities()
+        {
+            return _metadata.Values;
+        }
     }
 
     /// <summary>

MarkMpn.Sql4Cds.Engine/DataSource.cs

@@ -5,6 +5,8 @@
 using System.Collections.Generic;
 using static Microsoft.ApplicationInsights.MetricDimensionNames.TelemetryContext;
 using Microsoft.Crm.Sdk.Messages;
+using System.Linq;
+
 #if NETCOREAPP
 using Microsoft.PowerPlatform.Dataverse.Client;
 #else
@@ -87,6 +89,7 @@ public DataSource(IOrganizationService org, IAttributeMetadataCache metadata, IT
 
             JoinOperatorsAvailable = joinOperators;
             ColumnComparisonAvailable = version >= new Version("9.1.0.19251");
+            CrossTableColumnComparisonAvailable = version >= new Version("9.2");
             OrderByEntityNameAvailable = version >= new Version("9.1.0.25249");
         }
 
@@ -132,6 +135,11 @@ public DataSource()
         /// </summary>
         public virtual bool ColumnComparisonAvailable { get; }
 
+        /// <summary>
+        /// Indicates if the server supports column comparison conditions across tables in FetchXML
+        /// </summary>
+        public virtual bool CrossTableColumnComparisonAvailable { get; }
+
         /// <summary>
         /// Indicates if the server supports ordering by link-entities in FetchXML
         /// </summary>
@@ -206,7 +214,11 @@ private Collation LoadDefaultCollation()
             {
                 ColumnSet = new ColumnSet("localeid")
             };
-            var org = Connection.RetrieveMultiple(qry).Entities[0];
+            var org = Connection.RetrieveMultiple(qry).Entities.FirstOrDefault();
+
+            if (org == null)
+                return Collation.USEnglish;
+
             var lcid = org.GetAttributeValue<int>("localeid");
 
             // Collation options are set based on the default language. Most are CI/AI but a few are not
@@ -241,5 +253,10 @@ private Collation LoadDefaultCollation()
 
             return new Collation(lcid, !ci, !ai);
         }
+
+        public DataSource Clone()
+        {
+            return (DataSource)MemberwiseClone();
+        }
     }
 }

MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDataNode.cs

@@ -553,6 +553,10 @@ private bool TranslateFetchXMLCriteria(NodeCompilationContext context, DataSourc
                     if (!String.IsNullOrEmpty(attr2?.AttributeOf))
                         return false;
 
+                    // If we need to use cross-table column comparisons, check the server supports it
+                    if (!entityAlias.Equals(entityAlias2, StringComparison.OrdinalIgnoreCase) && !dataSource.CrossTableColumnComparisonAvailable)
+                        return false;
+
                     // We can use valueof="alias.attribute", but the alias of the root entity isn't visible. Swap the comparison round
                     // so that it can be added to the root entity and reference the value from the link entity.
                     if (!entityAlias.Equals(entityAlias2, StringComparison.OrdinalIgnoreCase) && entityAlias2.Equals(targetEntityAlias, StringComparison.OrdinalIgnoreCase))
@@ -596,8 +600,8 @@ private bool TranslateFetchXMLCriteria(NodeCompilationContext context, DataSourc
                                 },
                                 new condition
                                 {
-                                    entityname = StandardizeAlias(entityAlias, targetEntityAlias, items),
-                                    attribute = RemoveAttributeAlias(attrName2, entityAlias, targetEntityAlias, items),
+                                    entityname = StandardizeAlias(entityAlias2, targetEntityAlias, items),
+                                    attribute = RemoveAttributeAlias(attrName2, entityAlias2, targetEntityAlias, items),
                                     @operator = @operator.notnull
                                 }
                             }
@@ -607,7 +611,7 @@ private bool TranslateFetchXMLCriteria(NodeCompilationContext context, DataSourc
                     else if (op == @operator.eq && type == BooleanComparisonType.IsNotDistinctFrom)
                     {
                         // FetchXML eq operator does not match records where both fields are null.
-                        // This matches the logic for =, but IS DISTINCT FROM also allows nulls to match
+                        // This matches the logic for =, but IS NOT DISTINCT FROM also allows nulls to match
                         filter = new filter
                         {
                             type = filterType.or,
@@ -627,8 +631,8 @@ private bool TranslateFetchXMLCriteria(NodeCompilationContext context, DataSourc
                                         },
                                         new condition
                                         {
-                                            entityname = StandardizeAlias(entityAlias, targetEntityAlias, items),
-                                            attribute = RemoveAttributeAlias(attrName2, entityAlias, targetEntityAlias, items),
+                                            entityname = StandardizeAlias(entityAlias2, targetEntityAlias, items),
+                                            attribute = RemoveAttributeAlias(attrName2, entityAlias2, targetEntityAlias, items),
                                             @operator = @operator.@null
                                         }
                                     }

MarkMpn.Sql4Cds.Engine/ExecutionPlan/ExecuteMessageNode.cs

@@ -174,7 +174,7 @@ private void SetOutputSchema(DataSource dataSource, Message message, TSqlFragmen
             if (message.OutputParameters.All(f => f.IsScalarType()))
             {
                 foreach (var value in message.OutputParameters)
-                    AddSchemaColumn(value.Name, SqlTypeConverter.NetToSqlType(value.Type).ToSqlType(dataSource)); // TODO: How are OSV and ER fields represented?
+                    AddSchemaColumn(value.Name, value.GetSqlDataType(dataSource));
             }
             else
             {
@@ -206,16 +206,8 @@ private void SetOutputSchema(DataSource dataSource, Message message, TSqlFragmen
                     AddSchemaColumn(attrMetadata.LogicalName, attrMetadata.GetAttributeSqlType(dataSource, false));
 
                     // Add standard virtual attributes
-                    if (attrMetadata is EnumAttributeMetadata || attrMetadata is BooleanAttributeMetadata)
-                        AddSchemaColumn(attrMetadata.LogicalName + "name", DataTypeHelpers.NVarChar(FetchXmlScan.LabelMaxLength, dataSource.DefaultCollation, CollationLabel.CoercibleDefault));
-
-                    if (attrMetadata is LookupAttributeMetadata lookup)
-                    {
-                        AddSchemaColumn(attrMetadata.LogicalName + "name", DataTypeHelpers.NVarChar(lookup.Targets == null || lookup.Targets.Length == 0 ? 100 : lookup.Targets.Select(e => ((StringAttributeMetadata)dataSource.Metadata[e].Attributes.SingleOrDefault(a => a.LogicalName == dataSource.Metadata[e].PrimaryNameAttribute))?.MaxLength ?? 100).Max(), dataSource.DefaultCollation, CollationLabel.CoercibleDefault));
-
-                        if (lookup.Targets?.Length != 1 && lookup.AttributeType != AttributeTypeCode.PartyList)
-                            AddSchemaColumn(attrMetadata.LogicalName + "type", DataTypeHelpers.NVarChar(MetadataExtensions.EntityLogicalNameMaxLength, dataSource.DefaultCollation, CollationLabel.CoercibleDefault));
-                    }
+                    foreach (var virtualAttr in attrMetadata.GetVirtualAttributes(dataSource, false))
+                        AddSchemaColumn(attrMetadata.LogicalName + virtualAttr.Suffix, virtualAttr.DataType);
                 }
 
                 if (audit)
@@ -564,7 +556,7 @@ public static ExecuteMessageNode FromMessage(SchemaObjectFunctionTableReference
                 var f = expectedInputParameters[i];
                 var sourceExpression = tvf.Parameters[i];
                 sourceExpression.GetType(context, out var sourceType);
-                var expectedType = SqlTypeConverter.NetToSqlType(f.Type).ToSqlType(context.PrimaryDataSource);
+                var expectedType = f.GetSqlDataType(context.PrimaryDataSource);
 
                 if (!SqlTypeConverter.CanChangeTypeImplicit(sourceType, expectedType))
                     throw new NotSupportedQueryFragmentException(Sql4CdsError.TypeClash(tvf.Parameters[f.Position], sourceType, expectedType));
@@ -666,7 +658,7 @@ public static ExecuteMessageNode FromMessage(ExecutableProcedureReference sproc,
 
                 var sourceExpression = sproc.Parameters[i].ParameterValue;
                 sourceExpression.GetType(context, out var sourceType);
-                var expectedType = SqlTypeConverter.NetToSqlType(targetParam.Type).ToSqlType(context.PrimaryDataSource);
+                var expectedType = targetParam.GetSqlDataType(context.PrimaryDataSource);
 
                 if (!SqlTypeConverter.CanChangeTypeImplicit(sourceType, expectedType))
                 {

MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs

@@ -75,7 +75,6 @@ public InvalidPagingException(string message) : base(message)
             }
         }
 
-        public const int LabelMaxLength = 200;
         private static readonly XmlSerializer _serializer = new XmlSerializer(typeof(FetchXml.FetchType));
 
         private Dictionary<string, List<ParameterizedCondition>> _parameterizedConditions;
@@ -1290,21 +1289,8 @@ private void AddSchemaAttribute(DataSource dataSource, ColumnList schema, Dictio
                 return;
 
             // Add standard virtual attributes
-            if (attrMetadata is MultiSelectPicklistAttributeMetadata)
-                AddSchemaAttribute(schema, aliases, AddSuffix(fullName, "name"), (attrMetadata.LogicalName + "name").EscapeIdentifier(), DataTypeHelpers.NVarChar(Int32.MaxValue, dataSource.DefaultCollation, CollationLabel.Implicit), notNull);
-            else if (attrMetadata is EnumAttributeMetadata || attrMetadata is BooleanAttributeMetadata)
-                AddSchemaAttribute(schema, aliases, AddSuffix(fullName, "name"), (attrMetadata.LogicalName + "name").EscapeIdentifier(), DataTypeHelpers.NVarChar(LabelMaxLength, dataSource.DefaultCollation, CollationLabel.Implicit), notNull);
-
-            if (attrMetadata is LookupAttributeMetadata lookup)
-            {
-                AddSchemaAttribute(schema, aliases, AddSuffix(fullName, "name"), (attrMetadata.LogicalName + "name").EscapeIdentifier(), DataTypeHelpers.NVarChar(lookup.Targets == null || lookup.Targets.Length == 0 ? 100 : lookup.Targets.Select(e => (dataSource.Metadata[e].Attributes.SingleOrDefault(a => a.LogicalName == dataSource.Metadata[e].PrimaryNameAttribute) as StringAttributeMetadata)?.MaxLength ?? 100).Max(), dataSource.DefaultCollation, CollationLabel.Implicit), notNull);
-
-                if (lookup.Targets?.Length != 1 && lookup.AttributeType != AttributeTypeCode.PartyList)
-                    AddSchemaAttribute(schema, aliases, AddSuffix(fullName, "type"), (attrMetadata.LogicalName + "type").EscapeIdentifier(), DataTypeHelpers.NVarChar(MetadataExtensions.EntityLogicalNameMaxLength, dataSource.DefaultCollation, CollationLabel.Implicit), notNull); ;
-
-                if (lookup.Targets != null && lookup.Targets.Any(logicalName => dataSource.Metadata[logicalName].DataProviderId == DataProviders.ElasticDataProvider))
-                    AddSchemaAttribute(schema, aliases, AddSuffix(fullName, "pid"), (attrMetadata.LogicalName + "pid").EscapeIdentifier(), DataTypeHelpers.NVarChar(100, dataSource.DefaultCollation, CollationLabel.Implicit), false);
-            }
+            foreach (var virtualAttr in attrMetadata.GetVirtualAttributes(dataSource, false))
+                AddSchemaAttribute(schema, aliases, AddSuffix(fullName, virtualAttr.Suffix), (attrMetadata.LogicalName + virtualAttr.Suffix).EscapeIdentifier(), virtualAttr.DataType, virtualAttr.NotNull ?? notNull);
         }
 
         private void AddSchemaAttribute(ColumnList schema, Dictionary<string, IReadOnlyList<string>> aliases, string fullName, string simpleName, DataTypeReference type, bool notNull)

MarkMpn.Sql4Cds.Engine/ExecutionPlan/QueryExecutionException.cs

@@ -3,6 +3,7 @@
 using System.Linq;
 using System.ServiceModel;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Microsoft.SqlServer.TransactSql.ScriptDom;
 using Microsoft.Xrm.Sdk;
@@ -106,6 +107,10 @@ private static int FaultCodeToSqlError(OrganizationServiceFault fault)
                 }
             }
 
+            var match = Regex.Match(fault.Message, "Sql Number: (\\d+)");
+            if (match.Success)
+                return Int32.Parse(match.Groups[1].Value);
+
             return 10337;
         }
     }

MarkMpn.Sql4Cds.Engine/ExecutionPlan/RaiseErrorNode.cs

@@ -103,6 +103,8 @@ public void Execute(NodeExecutionContext context, out int recordsAffected, out s
         {
             try
             {
+                _executionCount++;
+
                 var ecc = new ExpressionCompilationContext(context, null, null);
                 var eec = new ExpressionExecutionContext(context);
 

MarkMpn.Sql4Cds.Engine/ExecutionPlan/ThrowNode.cs

@@ -94,6 +94,8 @@ public override IEnumerable<IExecutionPlanNode> GetSources()
 
         public void Execute(NodeExecutionContext context, out int recordsAffected, out string message)
         {
+            _executionCount++;
+
             if (ErrorNumber == null)
                 context.Log(context.Error);
 

MarkMpn.Sql4Cds.Engine/IAttributeMetadataCache.cs

@@ -1,4 +1,5 @@
-using Microsoft.Xrm.Sdk.Metadata;
+using System.Collections.Generic;
+using Microsoft.Xrm.Sdk.Metadata;
 
 namespace MarkMpn.Sql4Cds.Engine
 {
@@ -88,5 +89,11 @@ public interface IAttributeMetadataCache
         /// Returns a list of entity logical names that are enabled for recycle bin access
         /// </summary>
         string[] RecycleBinEntities { get; }
+
+        /// <summary>
+        /// Gets a list of all available entities for autocomplete
+        /// </summary>
+        /// <returns>A list of all available entities for autocomplete</returns>
+        IEnumerable<EntityMetadata> GetAllEntities();
     }
 }

MarkMpn.Sql4Cds.Engine/MessageCache.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using MarkMpn.Sql4Cds.Engine.ExecutionPlan;
 using Microsoft.Crm.Sdk.Messages;
 using Microsoft.Xrm.Sdk;
 using Microsoft.Xrm.Sdk.Metadata;
@@ -429,5 +430,10 @@ public bool IsScalarType()
 
             return false;
         }
+
+        /// <summary>
+        /// Returns the SQL data type for the parameter
+        /// </summary>
+        public Microsoft.SqlServer.TransactSql.ScriptDom.DataTypeReference GetSqlDataType(DataSource dataSource) => SqlTypeConverter.NetToSqlType(Type).ToSqlType(dataSource);
     }
 }

MarkMpn.Sql4Cds.Engine/MetaMetadataCache.cs

@@ -304,5 +304,11 @@ public bool TryGetMinimalData(string logicalName, out EntityMetadata metadata)
 
         /// <inheritdoc/>
         public string[] RecycleBinEntities => _inner.RecycleBinEntities;
+
+        /// <inheritdoc/>
+        public IEnumerable<EntityMetadata> GetAllEntities()
+        {
+            return _inner.GetAllEntities().Concat(_customMetadata.Values);
+        }
     }
 }