Cross instance

MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs

@@ -82,6 +82,8 @@ void IQueryExecutionOptions.RetrievingNextPage()
         {
         }
 
+        string IQueryExecutionOptions.PrimaryDataSource => "local";
+
         Guid IQueryExecutionOptions.UserId => Guid.NewGuid();
 
         [TestMethod]
@@ -2996,7 +2998,7 @@ public void CrossInstanceJoin()
                     TableSizeCache = new StubTableSizeCache()
                 }
             };
-            var planBuilder = new ExecutionPlanBuilder(datasources, "uat", this);
+            var planBuilder = new ExecutionPlanBuilder(datasources, this);
 
             var query = "SELECT uat.name, prod.name FROM uat.dbo.account AS uat INNER JOIN prod.dbo.account AS prod ON uat.accountid = prod.accountid WHERE uat.name <> prod.name AND uat.name LIKE '%test%'";
 

MarkMpn.Sql4Cds.Engine.Tests/Sql2FetchXmlTests.cs

@@ -52,6 +52,8 @@ void IQueryExecutionOptions.RetrievingNextPage()
         {
         }
 
+        string IQueryExecutionOptions.PrimaryDataSource => "local";
+
         Guid IQueryExecutionOptions.UserId => Guid.NewGuid();
 
         [TestMethod]
@@ -2981,6 +2983,52 @@ public void CastDateTimeToDate()
             Assert.AreEqual(new DateTime(2000, 1, 1), result.Entities[0].GetAttributeValue<DateTime>("converted"));
         }
 
+        [TestMethod]
+        public void GroupByPrimaryFunction()
+        {
+            var context = new XrmFakedContext();
+            context.InitializeMetadata(Assembly.GetExecutingAssembly());
+
+            var org = context.GetOrganizationService();
+            var metadata = new AttributeMetadataCache(org);
+            var sql2FetchXml = new Sql2FetchXml(metadata, true);
+
+            var query = "SELECT left(firstname, 1) AS initial, count(*) AS count FROM contact GROUP BY left(firstname, 1) ORDER BY 2 DESC";
+
+            var queries = sql2FetchXml.Convert(query);
+
+            var contact1 = Guid.NewGuid();
+            var contact2 = Guid.NewGuid();
+            var contact3 = Guid.NewGuid();
+
+            context.Data["contact"] = new Dictionary<Guid, Entity>
+            {
+                [contact1] = new Entity("contact", contact1)
+                {
+                    ["firstname"] = "Mark",
+                    ["contactid"] = contact1
+                },
+                [contact2] = new Entity("contact", contact2)
+                {
+                    ["firstname"] = "Matt",
+                    ["contactid"] = contact2
+                },
+                [contact3] = new Entity("contact", contact3)
+                {
+                    ["firstname"] = "Rich",
+                    ["contactid"] = contact3
+                }
+            };
+
+            var select = queries[0];
+            select.Execute(GetDataSources(context), this);
+            var result = (EntityCollection)select.Result;
+            Assert.AreEqual("M", result.Entities[0].GetAttributeValue<string>("initial"));
+            Assert.AreEqual(2, result.Entities[0].GetAttributeValue<int>("count"));
+            Assert.AreEqual("R", result.Entities[1].GetAttributeValue<string>("initial"));
+            Assert.AreEqual(1, result.Entities[1].GetAttributeValue<int>("count"));
+        }
+
         private void AssertFetchXml(Query[] queries, string fetchXml)
         {
             Assert.AreEqual(1, queries.Length);

MarkMpn.Sql4Cds.Engine.Tests/StubOptions.cs

@@ -64,6 +64,8 @@ void IQueryExecutionOptions.RetrievingNextPage()
         {
         }
 
+        string IQueryExecutionOptions.PrimaryDataSource => "local";
+
         Guid IQueryExecutionOptions.UserId => Guid.NewGuid();
     }
 }

MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDmlNode.cs

@@ -81,6 +81,8 @@ public void Dispose()
         /// <summary>
         /// The instance that this node will be executed against
         /// </summary>
+        [Category("Data Source")]
+        [Description("The data source this query is executed against")]
         public string DataSource { get; set; }
 
         /// <summary>
@@ -210,6 +212,7 @@ protected Dictionary<string, Func<Entity, object>> CompileColumnMappings(EntityM
                 var destSqlType = SqlTypeConverter.NetToSqlType(destType);
 
                 var expr = (Expression)Expression.Property(entityParam, typeof(Entity).GetCustomAttribute<DefaultMemberAttribute>().MemberName, Expression.Constant(sourceColumnName));
+                var originalExpr = expr;
 
                 if (sourceType == typeof(object))
                 {
@@ -228,27 +231,36 @@ protected Dictionary<string, Func<Entity, object>> CompileColumnMappings(EntityM
                         // Special case: intersect attributes can be simple guids
                         if (metadata.IsIntersect != true)
                         {
-                            Expression targetExpr;
-
-                            if (lookupAttr.Targets.Length == 1)
+                            if (sourceType == typeof(SqlEntityReference))
                             {
-                                targetExpr = Expression.Constant(lookupAttr.Targets[0]);
+                                expr = SqlTypeConverter.Convert(originalExpr, sourceType);
+                                convertedExpr = SqlTypeConverter.Convert(expr, typeof(EntityReference));
                             }
                             else
                             {
-                                var sourceTargetColumnName = mappings[destAttributeName + "type"];
-                                var sourceTargetType = schema.Schema[sourceTargetColumnName];
-                                targetExpr = Expression.Property(entityParam, typeof(Entity).GetCustomAttribute<DefaultMemberAttribute>().MemberName, Expression.Constant(sourceTargetColumnName));
-                                targetExpr = SqlTypeConverter.Convert(targetExpr, sourceTargetType);
-                                targetExpr = SqlTypeConverter.Convert(targetExpr, typeof(SqlString));
-                                targetExpr = SqlTypeConverter.Convert(targetExpr, typeof(string));
+                                Expression targetExpr;
+
+                                if (lookupAttr.Targets.Length == 1)
+                                {
+                                    targetExpr = Expression.Constant(lookupAttr.Targets[0]);
+                                }
+                                else
+                                {
+                                    var sourceTargetColumnName = mappings[destAttributeName + "type"];
+                                    var sourceTargetType = schema.Schema[sourceTargetColumnName];
+                                    targetExpr = Expression.Property(entityParam, typeof(Entity).GetCustomAttribute<DefaultMemberAttribute>().MemberName, Expression.Constant(sourceTargetColumnName));
+                                    targetExpr = SqlTypeConverter.Convert(targetExpr, sourceTargetType);
+                                    targetExpr = SqlTypeConverter.Convert(targetExpr, typeof(SqlString));
+                                    targetExpr = SqlTypeConverter.Convert(targetExpr, typeof(string));
+                                }
+
+                                convertedExpr = Expression.New(
+                                    typeof(EntityReference).GetConstructor(new[] { typeof(string), typeof(Guid) }),
+                                    targetExpr,
+                                    Expression.Convert(convertedExpr, typeof(Guid))
+                                );
                             }
 
-                            convertedExpr = Expression.New(
-                                typeof(EntityReference).GetConstructor(new[] { typeof(string), typeof(Guid) }),
-                                targetExpr,
-                                Expression.Convert(convertedExpr, typeof(Guid))
-                            );
                             destType = typeof(EntityReference);
                         }
                     }

MarkMpn.Sql4Cds.Engine/ExecutionPlan/BulkDeleteJobNode.cs

@@ -18,6 +18,8 @@ class BulkDeleteJobNode : BaseNode, IDmlQueryExecutionPlanNode, IFetchXmlExecuti
         private int _executionCount;
         private readonly Timer _timer = new Timer();
 
+        [Category("Data Source")]
+        [Description("The data source this query is executed against")]
         public string DataSource { get; set; }
 
         [Browsable(false)]

MarkMpn.Sql4Cds.Engine/ExecutionPlan/ExpressionExtensions.cs

@@ -246,7 +246,18 @@ cmp.SecondExpression is FunctionCall func
             var rhs = cmp.SecondExpression.ToExpression(schema, nonAggregateSchema, parameterTypes, entityParam, parameterParam, optionsParam);
 
             if (!SqlTypeConverter.CanMakeConsistentTypes(lhs.Type, rhs.Type, out var type))
-                throw new NotSupportedQueryFragmentException($"No implicit conversion exists for types {lhs} and {rhs}", cmp);
+            {
+                // Special case - we can filter on entity reference types by string
+                if (lhs.Type == typeof(SqlEntityReference) && rhs.Type == typeof(SqlString) ||
+                    lhs.Type == typeof(SqlString) && rhs.Type == typeof(SqlEntityReference))
+                {
+                    type = typeof(SqlGuid);
+                }
+                else
+                {
+                    throw new NotSupportedQueryFragmentException($"No implicit conversion exists for types {lhs.Type.Name} and {rhs.Type.Name}", cmp);
+                }
+            }
 
             if (lhs.Type != type)
                 lhs = SqlTypeConverter.Convert(lhs, type);
@@ -463,8 +474,29 @@ private static MethodInfo GetMethod(Type targetType, FunctionCall func, Type[] p
             if (correctParameterCount.Count > 1)
                 throw new NotSupportedQueryFragmentException("Ambiguous method", func);
 
-            // Check parameter types can be converted
+            var method = correctParameterCount[0].Method;
             var parameters = correctParameterCount[0].Parameters;
+
+            if (correctParameterCount[0].Method.IsGenericMethodDefinition)
+            {
+                // Create the generic method based on the type of the generic arguments
+                var genericArguments = correctParameterCount[0].Method.GetGenericArguments();
+                var genericArgumentValues = new Type[genericArguments.Length];
+
+                foreach (var param in correctParameterCount[0].Parameters)
+                {
+                    for (var i = 0; i < genericArguments.Length; i++)
+                    {
+                        if (param.ParameterType == genericArguments[i] && genericArgumentValues[i] == null)
+                            genericArgumentValues[i] = paramTypes[i];
+                    }
+                }
+
+                method = method.MakeGenericMethod(genericArgumentValues);
+                parameters = method.GetParameters();
+            }
+
+            // Check parameter types can be converted
             var paramOffset = targetType == typeof(FetchXmlConditionMethods) ? 1 : 0;
 
             for (var i = 0; i < parameters.Length; i++)
@@ -495,7 +527,7 @@ private static MethodInfo GetMethod(Type targetType, FunctionCall func, Type[] p
                     throw new NotSupportedQueryFragmentException($"Cannot convert {paramTypes[i]} to {paramType}", i < paramOffset ? func : func.Parameters[i - paramOffset]);
             }
 
-            return correctParameterCount[0].Method;
+            return method;
         }
 
         private static Expression ToExpression(this FunctionCall func, NodeSchema schema, NodeSchema nonAggregateSchema, IDictionary<string, Type> parameterTypes, ParameterExpression entityParam, ParameterExpression parameterParam, ParameterExpression optionsParam)
@@ -1165,9 +1197,9 @@ private static SqlDateTime GetCurrentTimestamp(IQueryExecutionOptions options)
                 return new SqlDateTime(DateTime.UtcNow);
         }
 
-        private static SqlGuid GetCurrentUser(IQueryExecutionOptions options)
+        private static SqlEntityReference GetCurrentUser(IQueryExecutionOptions options)
         {
-            return options.UserId;
+            return new SqlEntityReference(options.PrimaryDataSource, "systemuser", options.UserId);
         }
 
         /// <summary>

MarkMpn.Sql4Cds.Engine/ExecutionPlan/FetchXmlScan.cs

@@ -58,6 +58,7 @@ public void SetValue(object value)
 
         private Dictionary<string, ParameterizedCondition> _parameterizedConditions;
         private HashSet<string> _entityNameGroupings;
+        private Dictionary<string, string> _primaryKeyColumns;
 
         public FetchXmlScan()
         {
@@ -67,6 +68,8 @@ public FetchXmlScan()
         /// <summary>
         /// The instance that this node will be executed against
         /// </summary>
+        [Category("Data Source")]
+        [Description("The data source this query is executed against")]
         public string DataSource { get; set; }
 
         /// <summary>
@@ -274,16 +277,23 @@ private void OnRetrievedEntity(Entity entity, NodeSchema schema, IQueryExecution
                 if (!entity.Contains(attribute.Key + "type"))
                     entity[attribute.Key + "type"] = ((EntityReference)attribute.Value).LogicalName;
 
-                entity[attribute.Key] = ((EntityReference)attribute.Value).Id;
+                //entity[attribute.Key] = ((EntityReference)attribute.Value).Id;
             }
 
             // Convert values to SQL types
             foreach (var col in schema.Schema)
             {
+                object sqlValue;
+
                 if (entity.Attributes.TryGetValue(col.Key, out var value) && value != null)
-                    entity[col.Key] = SqlTypeConverter.NetToSqlType(value);
+                    sqlValue = SqlTypeConverter.NetToSqlType(DataSource, value);
                 else
-                    entity[col.Key] = SqlTypeConverter.GetNullValue(col.Value);
+                    sqlValue = SqlTypeConverter.GetNullValue(col.Value);
+
+                if (_primaryKeyColumns.TryGetValue(col.Key, out var logicalName) && sqlValue is SqlGuid guid)
+                    sqlValue = new SqlEntityReference(DataSource, logicalName, guid);
+
+                entity[col.Key] = sqlValue;
             }
         }
 
@@ -400,6 +410,7 @@ public override NodeSchema GetSchema(IDictionary<string, DataSource> dataSources
             if (!dataSources.TryGetValue(DataSource, out var dataSource))
                 throw new NotSupportedQueryFragmentException("Missing datasource " + DataSource);
 
+            _primaryKeyColumns = new Dictionary<string, string>();
             var schema = new NodeSchema();
 
             // Add each attribute from the main entity and recurse into link entities
@@ -631,6 +642,9 @@ private void AddSchemaAttribute(NodeSchema schema, string fullName, string simpl
             // Add the logical attribute
             AddSchemaAttribute(schema, fullName, simpleName, type);
 
+            if (attrMetadata.IsPrimaryId == true)
+                _primaryKeyColumns[fullName] = attrMetadata.EntityLogicalName;
+
             if (FetchXml.aggregate)
                 return;
 

MarkMpn.Sql4Cds.Engine/ExecutionPlan/GlobalOptionSetQueryNode.cs

@@ -70,6 +70,8 @@ static GlobalOptionSetQueryNode()
         /// <summary>
         /// The instance that this node will be executed against
         /// </summary>
+        [Category("Data Source")]
+        [Description("The data source this query is executed against")]
         public string DataSource { get; set; }
 
         /// <summary>

MarkMpn.Sql4Cds.Engine/ExecutionPlan/MetadataQueryNode.cs

@@ -138,6 +138,8 @@ static MetadataQueryNode()
         /// <summary>
         /// The instance that this node will be executed against
         /// </summary>
+        [Category("Data Source")]
+        [Description("The data source this query is executed against")]
         public string DataSource { get; set; }
 
         /// <summary>

MarkMpn.Sql4Cds.Engine/ExecutionPlan/RetrieveTotalRecordCountNode.cs

@@ -15,6 +15,8 @@ namespace MarkMpn.Sql4Cds.Engine.ExecutionPlan
     /// </summary>
     class RetrieveTotalRecordCountNode : BaseDataNode
     {
+        [Category("Data Source")]
+        [Description("The data source this query is executed against")]
         public string DataSource { get; set; }
 
         /// <summary>

MarkMpn.Sql4Cds.Engine/ExecutionPlan/SqlNode.cs

@@ -23,6 +23,8 @@ class SqlNode : BaseNode, IDataSetExecutionPlanNode
 
         public override TimeSpan Duration => _duration;
 
+        [Category("Data Source")]
+        [Description("The data source this query is executed against")]
         public string DataSource { get; set; }
 
         [Category("TDS Endpoint")]
@@ -74,8 +76,26 @@ public DataTable Execute(IDictionary<string, DataSource> dataSources, IQueryExec
                             adapter.Fill(result);
                         }
 
-                        var columnSqlTypes = result.Columns.Cast<DataColumn>().Select(col => SqlTypeConverter.NetToSqlType(col.DataType)).ToArray();
-                        var columnNullValues = columnSqlTypes.Select(type => SqlTypeConverter.GetNullValue(type)).ToArray();
+                        // SQL doesn't know the data type of NULL, so SELECT NULL will be returned with a schema type
+                        // of SqlInt32. This causes problems trying to convert it to other types for updates/inserts,
+                        // so change all-null columns to object
+                        // https://github.com/MarkMpn/Sql4Cds/issues/122
+                        var nullColumns = result.Columns
+                            .Cast<DataColumn>()
+                            .Select((col, colIndex) => result.Rows
+                                .Cast<DataRow>()
+                                .Select(row => DBNull.Value.Equals(row[colIndex]))
+                                .All(isNull => isNull)
+                                )
+                            .ToArray();
+
+                        var columnSqlTypes = result.Columns
+                            .Cast<DataColumn>()
+                            .Select((col, colIndex) => nullColumns[colIndex] ? typeof(object) : SqlTypeConverter.NetToSqlType(col.DataType))
+                            .ToArray();
+                        var columnNullValues = columnSqlTypes
+                            .Select(type => SqlTypeConverter.GetNullValue(type))
+                            .ToArray();
 
                         // Values will be stored as BCL types, convert them to SqlXxx types for consistency with IDataExecutionPlanNodes
                         var sqlTable = new DataTable();
@@ -89,7 +109,7 @@ public DataTable Execute(IDictionary<string, DataSource> dataSources, IQueryExec
 
                             for (var i = 0; i < result.Columns.Count; i++)
                             {
-                                var sqlValue = DBNull.Value.Equals(row[i]) ? columnNullValues[i] : SqlTypeConverter.NetToSqlType(row[i]);
+                                var sqlValue = DBNull.Value.Equals(row[i]) ? columnNullValues[i] : SqlTypeConverter.NetToSqlType(DataSource, row[i]);
                                 sqlRow[i] = sqlValue;
                             }
                         }