Merge pull request #172 from microsoft/features/publish-20200405-1-ALL

Microsoft CDM Release Version Update: 1.0.9

docs/SchemaViz.css

@@ -377,6 +377,7 @@
     color:mediumblue;
     padding: 3px;
     padding-left: 6px;
+    white-space: pre-wrap;
 }
 
 

docs/index.html

@@ -219,4 +219,4 @@
 
 </body>
 
-</html>
+</html>

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls.ALL.nuspec

@@ -5,7 +5,7 @@
   <metadata>
 
     <id>Microsoft.CommonDataModel.ObjectModel.Adapter.Adls.All</id>
-    <version>1.0.5</version>
+    <version>1.0.9</version>
     <description>The ADLS adapter implementation for the Microsoft Common Data Model Object Model.</description>
 
     <authors>Microsoft Corporation.</authors>

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <TargetFrameworks>netstandard2.0;net45;net462</TargetFrameworks>
-    <version>1.0.5</version>
+    <version>1.0.9</version>
     <Description>The ADLS adapter implementation for the Microsoft Common Data Model Object Model.</Description>
 
     <Authors>Microsoft Corporation</Authors>

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls.nuspec

@@ -5,7 +5,7 @@
   <metadata>
 
     <id>Microsoft.CommonDataModel.ObjectModel.Adapter.Adls</id>
-    <version>1.0.5</version>
+    <version>1.0.9</version>
     <description>The ADLS adapter implementation for the Microsoft Common Data Model Object Model.</description>
 
     <authors>Microsoft Corporation.</authors>

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Adapter.Adls/Storage/ADLSAdapter.cs

@@ -19,6 +19,7 @@ namespace Microsoft.CommonDataModel.ObjectModel.Storage
     using System.Security.Cryptography;
     using Newtonsoft.Json.Linq;
     using System.Diagnostics;
+    using Microsoft.CommonDataModel.ObjectModel.Utilities;
 
     public class ADLSAdapter : NetworkAdapter, StorageAdapter
     {
@@ -68,17 +69,22 @@ private set
         /// <summary>
         /// The client ID of an application accessing ADLS.
         /// </summary>
-        public string ClientId { get; private set; }
+        public string ClientId { get; set; }
 
         /// <summary>
         /// The secret for the app accessing ADLS.
         /// </summary>
-        public string Secret { get; private set; }
+        public string Secret { get; set; }
 
         /// <summary>
         /// The account/shared key.
         /// </summary>
-        public string SharedKey { get; private set; }
+        public string SharedKey { get; set; }
+
+        /// <summary>
+        /// The user-defined token provider.
+        /// </summary>
+        public TokenProvider TokenProvider { get; set; }
 
         /// <inheritdoc />
         public string LocationHint { get; set; }
@@ -123,11 +129,6 @@ private set
         /// </summary>
         private const string HttpXmsVersion = "x-ms-version";
 
-        /// <summary>
-        /// The user-defined token provider.
-        /// </summary>
-        private TokenProvider tokenProvider;
-
         internal const string Type = "adls";
 
         /// <summary>
@@ -170,7 +171,7 @@ public ADLSAdapter(string hostname, string root, TokenProvider tokenProvider)
         {
             this.Hostname = hostname;
             this.Root = root;
-            this.tokenProvider = tokenProvider;
+            this.TokenProvider = tokenProvider;
             this.httpClient = new CdmHttpClient();
         }
 
@@ -183,12 +184,14 @@ public bool CanRead()
         /// <inheritdoc />
         public async Task<string> ReadAsync(string corpusPath)
         {
-            String url = this.CreateAdapterPath(corpusPath);
-            var request = await this.BuildRequest(url, HttpMethod.Get);
+            string url = this.CreateAdapterPath(corpusPath);
 
-            var cdmResponse = await base.ExecuteRequest(request);
+            var httpRequest = await this.BuildRequest(url, HttpMethod.Get);
 
-            return await cdmResponse.Content.ReadAsStringAsync();
+            using (var cdmResponse = await base.ExecuteRequest(httpRequest))
+            {
+                return await cdmResponse.Content.ReadAsStringAsync();
+            }
         }
 
         /// <inheritdoc />
@@ -200,7 +203,7 @@ public bool CanWrite()
         /// <inheritdoc />
         public async Task WriteAsync(string corpusPath, string data)
         {
-            if (ensurePath($"{this.Root}{corpusPath}") == false)
+            if (EnsurePath($"{this.Root}{corpusPath}") == false)
             {
                 throw new Exception($"Could not create folder for document '{corpusPath}'");
             }
@@ -224,8 +227,12 @@ public async Task WriteAsync(string corpusPath, string data)
         public string CreateAdapterPath(string corpusPath)
         {
             var formattedCorpusPath = this.FormatCorpusPath(corpusPath);
+            if (formattedCorpusPath == null)
+            {
+                return null;
+            }
 
-            if(adapterPaths.ContainsKey(formattedCorpusPath))
+            if (adapterPaths.ContainsKey(formattedCorpusPath))
             {
                 return adapterPaths[formattedCorpusPath];
             } 
@@ -276,15 +283,16 @@ public void ClearCache()
         {
             var adapterPath = this.CreateAdapterPath(corpusPath);
 
-            var request = await this.BuildRequest(adapterPath, HttpMethod.Head);
+            var httpRequest = await this.BuildRequest(adapterPath, HttpMethod.Head);
 
             try
             {
-                CdmHttpResponse cdmResponse = await base.ExecuteRequest(request);
-
-                if (cdmResponse.StatusCode.Equals(HttpStatusCode.OK))
+                using (var cdmResponse = await base.ExecuteRequest(httpRequest))
                 {
-                    return cdmResponse.Content.Headers.LastModified;
+                    if (cdmResponse.StatusCode.Equals(HttpStatusCode.OK))
+                    {
+                        return cdmResponse.Content.Headers.LastModified;
+                    }
                 }
             }
             catch (HttpRequestException ex)
@@ -299,6 +307,11 @@ public void ClearCache()
         /// <inheritdoc />
         public async Task<List<string>> FetchAllFilesAsync(string folderCorpusPath)
         {
+            if (folderCorpusPath == null)
+            {
+                return null;
+            }
+
             var url = $"https://{this.Hostname}/{this.fileSystem}";
 
             var directory = $"{this.subPath}{FormatCorpusPath(folderCorpusPath)}";
@@ -308,10 +321,15 @@ public async Task<List<string>> FetchAllFilesAsync(string folderCorpusPath)
             }
 
             var request = await this.BuildRequest($"{url}?directory={directory}&recursive=True&resource=filesystem", HttpMethod.Get);
-            CdmHttpResponse cdmResponse = await base.ExecuteRequest(request);
 
-            if (cdmResponse.StatusCode.Equals(HttpStatusCode.OK))
+
+            using (var cdmResponse = await base.ExecuteRequest(request))
             {
+                if (!cdmResponse.StatusCode.Equals(HttpStatusCode.OK))
+                {
+                    return null;
+                }
+
                 string json = await cdmResponse.Content.ReadAsStringAsync();
                 JObject jObject1 = JObject.Parse(json);
 
@@ -331,11 +349,9 @@ public async Task<List<string>> FetchAllFilesAsync(string folderCorpusPath)
                         result.Add(this.FormatCorpusPath(nameWithoutSubPath));
                     }
                 }
-
+                
                 return result;
             }
-
-            return null;
         }
 
         /// <summary>
@@ -401,15 +417,15 @@ private Dictionary<string, string> ApplySharedKey(string sharedKey, string url,
             }
 
             // Hash the payload.
-            byte[] dataToHash = System.Text.Encoding.UTF8.GetBytes(builder.ToString().TrimEnd('\n'));
+            byte[] dataToHash = Encoding.UTF8.GetBytes(builder.ToString().TrimEnd('\n'));
             if (!TryFromBase64String(sharedKey, out byte[] bytes))
             {
                 throw new Exception("Couldn't encode the shared key.");
             }
 
             using (HMACSHA256 hmac = new HMACSHA256(bytes))
             {
-                string signedString = $"SharedKey {accountName}:{System.Convert.ToBase64String(hmac.ComputeHash(dataToHash))}";
+                string signedString = $"SharedKey {accountName}:{Convert.ToBase64String(hmac.ComputeHash(dataToHash))}";
 
                 headers.Add(HttpAuthorization, signedString);
             }
@@ -457,11 +473,15 @@ private void ExtractFilesystemAndSubPath(string root)
         /// <returns></returns>
         private string FormatCorpusPath(string corpusPath)
         {
-            if (corpusPath.StartsWith("adls:"))
+            var pathTuple = StorageUtils.SplitNamespacePath(corpusPath);
+            if (pathTuple == null)
             {
-                corpusPath = corpusPath.Substring(5);
+                return null;
             }
-            else if (corpusPath.Length > 0 && corpusPath[0] != '/')
+
+            corpusPath = pathTuple.Item2;
+
+            if (corpusPath.Length > 0 && corpusPath[0] != '/')
             {
                 corpusPath = $"/{corpusPath}";
             }
@@ -523,7 +543,7 @@ private Task<AuthenticationResult> GenerateBearerToken()
         /// <param name="content">The string content.</param>
         /// <param name="contentType">The content type.</param>
         /// <returns>The constructed Cdm Http request.</returns>
-        private async Task<CdmHttpRequest> BuildRequest(String url, HttpMethod method, string content = null, string contentType = null)
+        private async Task<CdmHttpRequest> BuildRequest(string url, HttpMethod method, string content = null, string contentType = null)
         {
             CdmHttpRequest request;
 
@@ -538,9 +558,9 @@ private async Task<CdmHttpRequest> BuildRequest(String url, HttpMethod method, s
 
                 request = this.SetUpCdmRequest(url, new Dictionary<string, string> { { "authorization", $"{token.AccessTokenType} {token.AccessToken}" } }, method);
             }
-            else if (this.tokenProvider != null)
+            else if (this.TokenProvider != null)
             {
-                request = this.SetUpCdmRequest(url, new Dictionary<string, string> { { "authorization", $"{this.tokenProvider.GetToken()}" } }, method);
+                request = this.SetUpCdmRequest(url, new Dictionary<string, string> { { "authorization", $"{this.TokenProvider.GetToken()}" } }, method);
             }
             else
             {
@@ -556,7 +576,7 @@ private async Task<CdmHttpRequest> BuildRequest(String url, HttpMethod method, s
             return request;
         }
 
-        private bool ensurePath(string pathFor)
+        private bool EnsurePath(string pathFor)
         {
             if (pathFor.LastIndexOf("/") == -1)
                 return false;

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Cdm/CdmTraitDefinition/CdmTraitDefinitionTest.cs

@@ -0,0 +1,30 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+namespace Microsoft.CommonDataModel.ObjectModel.Tests.Cdm
+{
+    using Microsoft.CommonDataModel.ObjectModel.Cdm;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class CdmTraitDefinitionTests
+    {
+        [TestMethod]
+        public void TestExtendsTraitPropertyOptional()
+        {
+            var corpus = new CdmCorpusDefinition();
+            var extendTraitRef1 = new CdmTraitReference(corpus.Ctx, "testExtendTraitName1", true, false);
+            var extendTraitRef2 = new CdmTraitReference(corpus.Ctx, "testExtendTraitName2", true, false);
+            var traitDefinition = new CdmTraitDefinition(corpus.Ctx, "testTraitName", extendTraitRef1);
+
+            Assert.AreEqual(extendTraitRef1, traitDefinition.ExtendsTrait);
+            traitDefinition.ExtendsTrait = null;
+            Assert.IsNull(traitDefinition.ExtendsTrait);
+
+            traitDefinition.ExtendsTrait = extendTraitRef2;
+            Assert.AreEqual(extendTraitRef2, traitDefinition.ExtendsTrait);
+            traitDefinition.ExtendsTrait = null;
+            Assert.IsNull(traitDefinition.ExtendsTrait);
+        }
+    }
+}

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Cdm/CorpusTests.cs

@@ -0,0 +1,59 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+namespace Microsoft.CommonDataModel.ObjectModel.Tests.Cdm
+{
+    using Microsoft.CommonDataModel.ObjectModel.Cdm;
+    using Microsoft.CommonDataModel.ObjectModel.Utilities;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+    using System.IO;
+    using System.Threading.Tasks;
+
+    /// <summary>
+    /// Test methods for the CdmCorpusDefinition class.
+    /// </summary>
+    [TestClass]
+    public class CorpusTests
+    {
+        private readonly string testsSubpath = Path.Combine("Cdm", "Corpus");
+
+        /// <summary>
+        /// Tests if a symbol imported with a moniker can be found as the last resource.
+        /// When resolving symbolEntity with respect to wrtEntity, the symbol fromEntity should be found correctly.
+        /// </summary>
+        [TestMethod]
+        public async Task TestResolveSymbolReference()
+        {
+            var corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestResolveSymbolReference");
+            corpus.SetEventCallback(new EventCallback
+            {
+                Invoke = (CdmStatusLevel statusLevel, string message) =>
+                {
+                    Assert.Fail(message);
+                }
+            }, CdmStatusLevel.Warning);
+
+            var wrtEntity = await corpus.FetchObjectAsync<CdmEntityDefinition>("local:/wrtEntity.cdm.json/wrtEntity");
+            var resOpt = new ResolveOptions(wrtEntity, new AttributeResolutionDirectiveSet());
+            await wrtEntity.CreateResolvedEntityAsync("NewEntity", resOpt);
+        }
+
+        /// <summary>
+        /// Tests if ComputeLastModifiedTimeAsync doesn't log errors related to reference validation.
+        /// </summary>
+        [TestMethod]
+        public async Task TestComputeLastModifiedTimeAsync()
+        {
+            var corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestComputeLastModifiedTimeAsync");
+            corpus.SetEventCallback(new EventCallback
+            {
+                Invoke = (level, message) =>
+                {
+                    Assert.Fail(message);
+                }
+            }, CdmStatusLevel.Error);
+
+            await corpus.ComputeLastModifiedTimeAsync("local:/default.manifest.cdm.json");
+        }
+    }
+}

objectModel/CSharp/Microsoft.CommonDataModel.ObjectModel.Tests/Cdm/DataPartitionPattern/DataPartitionPatternTest.cs

@@ -136,6 +136,32 @@ public async Task TestPatternWithNonExistingFolder()
             Assert.IsNotNull(cdmManifest.Entities[0].DataPartitionPatterns[0].LastFileStatusCheckTime);
         }
 
+        /// <summary>
+        /// Testing that patterns behave correctly with variations to rootLocation
+        /// </summary>
+        [TestMethod]
+        public async Task TestVariationsInRootLocation()
+        {
+            CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestVariationsInRootLocation");
+            CdmManifestDefinition manifest = await corpus.FetchObjectAsync<CdmManifestDefinition>("pattern.manifest.cdm.json");
+            await manifest.FileStatusCheckAsync();
+
+            CdmLocalEntityDeclarationDefinition startsWithSlash = manifest.Entities[0] as CdmLocalEntityDeclarationDefinition;
+            Assert.AreEqual(".*testfile.csv", startsWithSlash.DataPartitionPatterns[0].RegularExpression);
+            Assert.AreEqual(1, startsWithSlash.DataPartitions.Count);
+            Assert.AreEqual("/partitions/testfile.csv", startsWithSlash.DataPartitions[0].Location);
+
+            CdmLocalEntityDeclarationDefinition endsWithSlash = manifest.Entities[1] as CdmLocalEntityDeclarationDefinition;
+            Assert.AreEqual(".*testfile.csv", endsWithSlash.DataPartitionPatterns[0].RegularExpression);
+            Assert.AreEqual(1, endsWithSlash.DataPartitions.Count);
+            Assert.AreEqual("partitions/testfile.csv", endsWithSlash.DataPartitions[0].Location);
+
+            CdmLocalEntityDeclarationDefinition noSlash = manifest.Entities[2] as CdmLocalEntityDeclarationDefinition;
+            Assert.AreEqual(".*testfile.csv", noSlash.DataPartitionPatterns[0].RegularExpression);
+            Assert.AreEqual(1, noSlash.DataPartitions.Count);
+            Assert.AreEqual("partitions/testfile.csv", noSlash.DataPartitions[0].Location);
+        }
+
         /// <summary>
         /// Testing data partition patterns that use glob patterns
         /// </summary>