Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/master' into cte
Browse files Browse the repository at this point in the history
  • Loading branch information
@MarkMpn
MarkMpn committed Sep 17, 2023
2 parents e43617f + 9f577ba commit ca8c03f
Show file tree
Hide file tree
Showing 23 changed files with 808 additions and 273 deletions.
9 changes: 8 additions & 1 deletion AzureDataStudioExtension/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,13 @@
# Change Log

## [v7.5.1](https://github.com/MarkMpn/Sql4Cds/releases/tag/v7.5.0) - 2023-09-10
## [v7.5.2](https://github.com/MarkMpn/Sql4Cds/releases/tag/v7.5.2) - 2023-09-17

Fixed ordering of hash join results
Improved filter folding across joins
Improved handling of alias names requiring escaping
Improved display of column sets in properties window

## [v7.5.1](https://github.com/MarkMpn/Sql4Cds/releases/tag/v7.5.1) - 2023-09-10

Return correct schema for global option set values
Fixed applying alias to primary key field
Expand Down
21 changes: 21 additions & 0 deletions MarkMpn.Sql4Cds.Engine.Tests/AdoProviderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1493,5 +1493,26 @@ DECLARE @id EntityReference
}
}
}

[TestMethod]
public void ComplexFetchXmlAlias()
{
using (var con = new Sql4CdsConnection(_localDataSource))
using (var cmd = con.CreateCommand())
{
cmd.CommandText = "INSERT INTO account (name) VALUES ('Data8')";
cmd.ExecuteNonQuery();

cmd.CommandText = "SELECT name AS [acc. name] FROM account AS [acc. table]";

using (var reader = cmd.ExecuteReader())
{
Assert.IsTrue(reader.Read());
Assert.AreEqual("acc. name", reader.GetName(0));
Assert.AreEqual("Data8", reader.GetString(0));
Assert.IsFalse(reader.Read());
}
}
}
}
}
127 changes: 127 additions & 0 deletions MarkMpn.Sql4Cds.Engine.Tests/ExecutionPlanTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6176,5 +6176,132 @@ systemuser AS s
</entity>
</fetch>");
}

[TestMethod]
public void ComplexFetchXmlAlias()
{
var planBuilder = new ExecutionPlanBuilder(_dataSources.Values, new OptionsWrapper(this) { PrimaryDataSource = "uat" });

var query = "SELECT name FROM account AS [acc. table]";

var plans = planBuilder.Build(query, null, out _);

var select = AssertNode<SelectNode>(plans[0]);
Assert.AreEqual("[acc. table].name", select.ColumnSet[0].SourceColumn);

var fetch = AssertNode<FetchXmlScan>(select.Source);
Assert.AreEqual("acc. table", fetch.Alias);
}

[TestMethod]
public void ComplexMetadataAlias()
{
var planBuilder = new ExecutionPlanBuilder(_dataSources.Values, new OptionsWrapper(this) { PrimaryDataSource = "uat" });

var query = "SELECT logicalname FROM metadata.entity AS [m.d. table] WHERE [m.d. table].logicalname = 'account'";

var plans = planBuilder.Build(query, null, out _);

var select = AssertNode<SelectNode>(plans[0]);
Assert.AreEqual("[m.d. table].logicalname", select.ColumnSet[0].SourceColumn);

var metadata = AssertNode<MetadataQueryNode>(select.Source);
Assert.AreEqual("m.d. table", metadata.EntityAlias);
Assert.AreEqual(nameof(EntityMetadata.LogicalName), metadata.Query.Criteria.Conditions[0].PropertyName);
Assert.AreEqual(MetadataConditionOperator.Equals, metadata.Query.Criteria.Conditions[0].ConditionOperator);
Assert.AreEqual("account", metadata.Query.Criteria.Conditions[0].Value);
}

[TestMethod]
public void ComplexInlineTableAlias()
{
var planBuilder = new ExecutionPlanBuilder(_dataSources.Values, new OptionsWrapper(this) { PrimaryDataSource = "uat" });

var query = "SELECT [full name] FROM (VALUES ('Mark Carrington')) AS [inline table] ([full name])";

var plans = planBuilder.Build(query, null, out _);

var select = AssertNode<SelectNode>(plans[0]);
Assert.AreEqual("[inline table].[full name]", select.ColumnSet[0].SourceColumn);

var constant = AssertNode<ConstantScanNode>(select.Source);
Assert.AreEqual("inline table", constant.Alias);
Assert.AreEqual(1, constant.Schema.Count);
Assert.AreEqual(DataTypeHelpers.VarChar(15, Collation.USEnglish, CollationLabel.CoercibleDefault), constant.Schema["[full name]"].Type, DataTypeComparer.Instance);
}

[TestMethod]
public void FoldFiltersToUnionAllAndJoins()
{
var planBuilder = new ExecutionPlanBuilder(_dataSources.Values, new OptionsWrapper(this) { PrimaryDataSource = "uat" });

var query = @"
SELECT [union. all].eln,
[union. all].logicalname,
[union. all].environment
FROM (SELECT entitylogicalname AS eln,
logicalname,
'env1' AS environment
FROM uat.metadata.attribute
UNION ALL
SELECT entitylogicalname,
logicalname,
'env2' AS environment
FROM prod.metadata.attribute) AS [union. all]
INNER JOIN
french.metadata.attribute AS a2
ON [union. all].eln = a2.entitylogicalname
AND [union. all].logicalname = a2.logicalname
WHERE [union. all].eln IN ('systemuser', 'businessunit')
AND [union. all].logicalname IN ('createdon')
ORDER BY [union. all].eln";


var plans = planBuilder.Build(query, null, out _);

var select = AssertNode<SelectNode>(plans[0]);

var sort = AssertNode<SortNode>(select.Source);

var join1 = AssertNode<HashJoinNode>(sort.Source);
Assert.AreEqual("a2.entitylogicalname", join1.LeftAttribute.ToSql());
Assert.AreEqual("[union. all].eln", join1.RightAttribute.ToSql());
Assert.AreEqual("[union. all].logicalname = a2.logicalname", join1.AdditionalJoinCriteria.ToSql());

var mq1 = AssertNode<MetadataQueryNode>(join1.LeftSource);
Assert.AreEqual("french", mq1.DataSource);
Assert.AreEqual(nameof(EntityMetadata.LogicalName), mq1.Query.Criteria.Conditions[0].PropertyName);
Assert.AreEqual(MetadataConditionOperator.In, mq1.Query.Criteria.Conditions[0].ConditionOperator);
CollectionAssert.AreEqual(new[] { "systemuser", "businessunit" }, (string[])mq1.Query.Criteria.Conditions[0].Value);
Assert.AreEqual(nameof(AttributeMetadata.LogicalName), mq1.Query.AttributeQuery.Criteria.Conditions[0].PropertyName);
Assert.AreEqual(MetadataConditionOperator.In, mq1.Query.AttributeQuery.Criteria.Conditions[0].ConditionOperator);
CollectionAssert.AreEqual(new[] { "createdon" }, (string[])mq1.Query.AttributeQuery.Criteria.Conditions[0].Value);

var alias = AssertNode<AliasNode>(join1.RightSource);
Assert.AreEqual("union. all", alias.Alias);
CollectionAssert.AreEqual(new[] { "eln", "logicalname", "environment" }, alias.ColumnSet.Select(col => col.OutputColumn).ToArray());

var concat = AssertNode<ConcatenateNode>(alias.Source);

var compute2 = AssertNode<ComputeScalarNode>(concat.Sources[0]);
var mq2 = AssertNode<MetadataQueryNode>(compute2.Source);
Assert.AreEqual("uat", mq2.DataSource);
Assert.AreEqual(nameof(EntityMetadata.LogicalName), mq2.Query.Criteria.Conditions[0].PropertyName);
Assert.AreEqual(MetadataConditionOperator.In, mq2.Query.Criteria.Conditions[0].ConditionOperator);
CollectionAssert.AreEqual(new[] { "systemuser", "businessunit" }, (string[])mq2.Query.Criteria.Conditions[0].Value);
Assert.AreEqual(nameof(AttributeMetadata.LogicalName), mq2.Query.AttributeQuery.Criteria.Conditions[0].PropertyName);
Assert.AreEqual(MetadataConditionOperator.In, mq2.Query.AttributeQuery.Criteria.Conditions[0].ConditionOperator);
CollectionAssert.AreEqual(new[] { "createdon" }, (string[])mq2.Query.AttributeQuery.Criteria.Conditions[0].Value);

var compute3 = AssertNode<ComputeScalarNode>(concat.Sources[1]);
var mq3 = AssertNode<MetadataQueryNode>(compute3.Source);
Assert.AreEqual("prod", mq3.DataSource);
Assert.AreEqual(nameof(EntityMetadata.LogicalName), mq3.Query.Criteria.Conditions[0].PropertyName);
Assert.AreEqual(MetadataConditionOperator.In, mq3.Query.Criteria.Conditions[0].ConditionOperator);
CollectionAssert.AreEqual(new[] { "systemuser", "businessunit" }, (string[])mq3.Query.Criteria.Conditions[0].Value);
Assert.AreEqual(nameof(AttributeMetadata.LogicalName), mq3.Query.AttributeQuery.Criteria.Conditions[0].PropertyName);
Assert.AreEqual(MetadataConditionOperator.In, mq3.Query.AttributeQuery.Criteria.Conditions[0].ConditionOperator);
CollectionAssert.AreEqual(new[] { "createdon" }, (string[])mq3.Query.AttributeQuery.Criteria.Conditions[0].Value);
}
}
}
63 changes: 36 additions & 27 deletions MarkMpn.Sql4Cds.Engine/ExecutionPlan/AliasNode.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ public AliasNode(SelectNode select, Identifier identifier, NodeCompilationContex
{
if (string.IsNullOrEmpty(col.OutputColumn))
col.OutputColumn = context.GetExpressionName();
else
col.OutputColumn = col.OutputColumn.EscapeIdentifier();
}
}

Expand Down Expand Up @@ -77,28 +79,32 @@ private AliasNode()
public override void AddRequiredColumns(NodeCompilationContext context, IList<string> requiredColumns)
{
var mappings = ColumnSet.Where(col => !col.AllColumns).ToDictionary(col => col.OutputColumn, col => col.SourceColumn);
ColumnSet.Clear();
var required = new HashSet<string>(StringComparer.OrdinalIgnoreCase);

var escapedAlias = Alias.EscapeIdentifier();

// Map the aliased names to the base names
for (var i = 0; i < requiredColumns.Count; i++)
{
if (requiredColumns[i].StartsWith(Alias + "."))
if (requiredColumns[i].StartsWith(escapedAlias + "."))
{
requiredColumns[i] = requiredColumns[i].Substring(Alias.Length + 1);

if (!mappings.TryGetValue(requiredColumns[i], out var sourceCol))
sourceCol = requiredColumns[i];
requiredColumns[i] = requiredColumns[i].Substring(escapedAlias.Length + 1);

ColumnSet.Add(new SelectColumn
if (mappings.TryGetValue(requiredColumns[i], out var sourceCol))
{
SourceColumn = sourceCol,
OutputColumn = requiredColumns[i]
});

requiredColumns[i] = sourceCol;
required.Add(requiredColumns[i]);
requiredColumns[i] = sourceCol;
}
}
}

// Remove any unsued column mappings
for (var i = ColumnSet.Count - 1; i >= 0; i--)
{
if (!ColumnSet[i].AllColumns && !required.Contains(ColumnSet[i].OutputColumn))
ColumnSet.RemoveAt(i);
}

Source.AddRequiredColumns(context, requiredColumns);
}

Expand All @@ -120,7 +126,7 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext
{
// Remove any unused columns
var unusedColumns = constant.Schema.Keys
.Where(sourceCol => !ColumnSet.Any(col => col.SourceColumn.Split('.').Last() == sourceCol))
.Where(sourceCol => !ColumnSet.Any(col => col.SourceColumn.SplitMultiPartIdentifier().Last().EscapeIdentifier() == sourceCol))
.ToList();

foreach (var col in unusedColumns)
Expand All @@ -134,10 +140,10 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext
// Copy/rename any columns using the new aliases
foreach (var col in ColumnSet)
{
var sourceColumn = col.SourceColumn.Split('.').Last();
var sourceColumn = col.SourceColumn.SplitMultiPartIdentifier().Last();

if (String.IsNullOrEmpty(constant.Alias) && col.OutputColumn != col.SourceColumn ||
!String.IsNullOrEmpty(constant.Alias) && col.OutputColumn != constant.Alias + "." + col.SourceColumn)
!String.IsNullOrEmpty(constant.Alias) && col.OutputColumn != constant.Alias.EscapeIdentifier() + "." + col.SourceColumn)
{
constant.Schema[col.OutputColumn] = constant.Schema[sourceColumn];

Expand All @@ -157,18 +163,20 @@ public override IDataExecutionPlanNodeInternal FoldQuery(NodeCompilationContext
internal void FoldToFetchXML(FetchXmlScan fetchXml)
{
// Add the mappings to the FetchXML to produce the columns with the expected names, and hide all other possible columns
var originalAlias = fetchXml.Alias;
var originalAlias = fetchXml.Alias.EscapeIdentifier();
fetchXml.Alias = Alias;

var escapedAlias = Alias.EscapeIdentifier();

foreach (var col in ColumnSet)
{
if (col.SourceColumn != null && col.SourceColumn.StartsWith(originalAlias + "."))
col.SourceColumn = Alias + col.SourceColumn.Substring(originalAlias.Length);
col.SourceColumn = escapedAlias + col.SourceColumn.Substring(originalAlias.Length);

if (col.AllColumns)
col.OutputColumn = Alias;
col.OutputColumn = escapedAlias;
else if (col.OutputColumn != null)
col.OutputColumn = Alias + "." + col.OutputColumn;
col.OutputColumn = escapedAlias + "." + col.OutputColumn;

fetchXml.ColumnMappings.Add(col);
}
Expand All @@ -187,6 +195,7 @@ public override INodeSchema GetSchema(NodeCompilationContext context)
var aliases = new Dictionary<string, IReadOnlyList<string>>();
var primaryKey = (string)null;
var mappings = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var escapedAlias = Alias.EscapeIdentifier();

foreach (var col in ColumnSet)
{
Expand All @@ -197,15 +206,13 @@ public override INodeSchema GetSchema(NodeCompilationContext context)
if (col.SourceColumn != null && !sourceCol.Key.StartsWith(col.SourceColumn + "."))
continue;

var simpleName = sourceCol.Key.Split('.').Last();
var outputName = $"{Alias}.{simpleName}";

AddSchemaColumn(simpleName, sourceCol.Key, schema, aliases, ref primaryKey, mappings, sourceSchema);
var simpleName = sourceCol.Key.SplitMultiPartIdentifier().Last();
AddSchemaColumn(escapedAlias, simpleName, sourceCol.Key, schema, aliases, ref primaryKey, mappings, sourceSchema);
}
}
else
{
AddSchemaColumn(col.OutputColumn, col.SourceColumn, schema, aliases, ref primaryKey, mappings, sourceSchema);
AddSchemaColumn(escapedAlias, col.OutputColumn, col.SourceColumn, schema, aliases, ref primaryKey, mappings, sourceSchema);
}
}

Expand All @@ -231,12 +238,12 @@ public override INodeSchema GetSchema(NodeCompilationContext context)
sortOrder: sortOrder);
}

private void AddSchemaColumn(string outputColumn, string sourceColumn, ColumnList schema, Dictionary<string, IReadOnlyList<string>> aliases, ref string primaryKey, Dictionary<string, string> mappings, INodeSchema sourceSchema)
private void AddSchemaColumn(string escapedAlias, string outputColumn, string sourceColumn, ColumnList schema, Dictionary<string, IReadOnlyList<string>> aliases, ref string primaryKey, Dictionary<string, string> mappings, INodeSchema sourceSchema)
{
if (!sourceSchema.ContainsColumn(sourceColumn, out var normalized))
return;

var mapped = $"{Alias}.{outputColumn}";
var mapped = $"{escapedAlias}.{outputColumn}";
schema[mapped] = new ColumnDefinition(sourceSchema.Schema[normalized].Type, sourceSchema.Schema[normalized].IsNullable, false);
mappings[normalized] = mapped;

Expand All @@ -254,11 +261,13 @@ private void AddSchemaColumn(string outputColumn, string sourceColumn, ColumnLis

protected override IEnumerable<Entity> ExecuteInternal(NodeExecutionContext context)
{
var escapedAlias = Alias.EscapeIdentifier();

foreach (var entity in Source.Execute(context))
{
foreach (var col in ColumnSet)
{
var mapped = $"{Alias}.{col.OutputColumn}";
var mapped = $"{escapedAlias}.{col.OutputColumn.EscapeIdentifier()}";
entity[mapped] = entity[col.SourceColumn];
}

Expand Down
12 changes: 6 additions & 6 deletions MarkMpn.Sql4Cds.Engine/ExecutionPlan/BaseDataNode.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -458,7 +458,7 @@ private bool TranslateFetchXMLCriteria(NodeCompilationContext context, IAttribut
if (!schema.ContainsColumn(columnName, out columnName))
return false;

var parts = columnName.Split('.');
var parts = columnName.SplitMultiPartIdentifier();

if (parts.Length != 2)
return false;
Expand Down Expand Up @@ -487,7 +487,7 @@ private bool TranslateFetchXMLCriteria(NodeCompilationContext context, IAttribut
if (!schema.ContainsColumn(columnName2, out columnName2))
return false;

var parts2 = columnName2.Split('.');
var parts2 = columnName2.SplitMultiPartIdentifier();
var entityAlias2 = parts2[0];
var attrName2 = parts2[1];

Expand Down Expand Up @@ -592,7 +592,7 @@ private bool TranslateFetchXMLCriteria(NodeCompilationContext context, IAttribut
if (!schema.ContainsColumn(columnName, out columnName))
return false;

var parts = columnName.Split('.');
var parts = columnName.SplitMultiPartIdentifier();
var entityAlias = parts[0];
var attrName = parts[1];

Expand All @@ -619,7 +619,7 @@ private bool TranslateFetchXMLCriteria(NodeCompilationContext context, IAttribut
if (!schema.ContainsColumn(columnName, out columnName))
return false;

var parts = columnName.Split('.');
var parts = columnName.SplitMultiPartIdentifier();
var entityAlias = parts[0];
var attrName = parts[1];

Expand Down Expand Up @@ -652,7 +652,7 @@ private bool TranslateFetchXMLCriteria(NodeCompilationContext context, IAttribut
if (!schema.ContainsColumn(columnName, out columnName))
return false;

var parts = columnName.Split('.');
var parts = columnName.SplitMultiPartIdentifier();
var entityAlias = parts[0];
var attrName = parts[1];

Expand Down Expand Up @@ -699,7 +699,7 @@ private bool TranslateFetchXMLCriteria(NodeCompilationContext context, IAttribut
if (!schema.ContainsColumn(columnName, out columnName))
return false;

var parts = columnName.Split('.');
var parts = columnName.SplitMultiPartIdentifier();
var entityAlias = parts[0];
var attrName = parts[1];

Expand Down
1 change: 1 addition & 0 deletions MarkMpn.Sql4Cds.Engine/ExecutionPlan/ConcatenateNode.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,7 @@ class ConcatenateColumn
/// The name of the column that is generated in the output
/// </summary>
[Description("The name of the column that is generated in the output")]
[DictionaryKey]
public string OutputColumn { get; set; }

/// <summary>
Expand Down
Loading

0 comments on commit ca8c03f

Please sign in to comment.