Compute blob upfront

src/Compilers/CSharp/Portable/Compilation/CSharpCompilation.cs

@@ -2415,17 +2415,13 @@ private bool StartSourceChecksumCalculation(PEModuleBuilder moduleBeingBuilt, Di
 
                 foreach (var text in moduleBeingBuilt.EmbeddedTextsOpt)
                 {
-                    if (!text.IsDefault)
-                    {
-                        Debug.Assert(!string.IsNullOrEmpty(text.FilePath));
-                        Debug.Assert(text.Text?.Encoding != null);
+                    Debug.Assert(!string.IsNullOrEmpty(text.FilePath));
 
-                        string normalizedPath = moduleBeingBuilt.NormalizeDebugDocumentPath(text.FilePath, basePath: null);
-                        var existingDoc = moduleBeingBuilt.TryGetDebugDocumentForNormalizedPath(normalizedPath);
-                        if (existingDoc == null)
-                        {
-                            moduleBeingBuilt.AddDebugDocument(MakeDebugSourceDoucmentForEmbeddedText(normalizedPath, text));
-                        }
+                    string normalizedPath = moduleBeingBuilt.NormalizeDebugDocumentPath(text.FilePath, basePath: null);
+                    var existingDoc = moduleBeingBuilt.TryGetDebugDocumentForNormalizedPath(normalizedPath);
+                    if (existingDoc == null)
+                    {
+                        moduleBeingBuilt.AddDebugDocument(MakeDebugSourceDoucmentForEmbeddedText(normalizedPath, text));
                     }
                 }
 
@@ -2573,7 +2569,7 @@ private static void AddDebugSourceDocumentsForChecksumDirectives(
                     if (ChecksumMatches(checksumText, sourceInfo.Checksum))
                     {
                         var guid = Guid.Parse(checksumDirective.Guid.ValueText);
-                        if (guid == sourceInfo.AlgorithmId)
+                        if (guid == sourceInfo.ChecksumAlgorithmId)
                         {
                             // all parts match, nothing to do
                             continue;

src/Compilers/Core/CodeAnalysisTest/Text/LargeTextTests.cs

@@ -26,7 +26,7 @@ private static SourceText CreateSourceText(string s, Encoding encoding = null)
 
         private static SourceText CreateSourceText(Stream stream, Encoding encoding = null)
         {
-            return LargeText.Decode(stream, encoding ?? Encoding.UTF8, SourceHashAlgorithm.Sha1, throwIfBinaryDetected: true);
+            return LargeText.Decode(stream, encoding ?? Encoding.UTF8, SourceHashAlgorithm.Sha1, throwIfBinaryDetected: true, canBeEmbedded: false);
         }
 
         private const string HelloWorld = "Hello, world!";

src/Compilers/Core/CodeAnalysisTest/Text/SourceTextTests.cs

@@ -124,12 +124,12 @@ public void ChecksumAndBOM()
             VerifyChecksum(SourceText.From(streamBOM, encodingBOM, checksumAlgorigthm), checksumBOM);
 
             // LargeText from stream no BOM. Checksum should ignore explicit encoding.
-            VerifyChecksum(LargeText.Decode(streamNoBOM, encodingNoBOM, checksumAlgorigthm, throwIfBinaryDetected: false), checksumNoBOM);
-            VerifyChecksum(LargeText.Decode(streamNoBOM, encodingBOM, checksumAlgorigthm, throwIfBinaryDetected: false), checksumNoBOM);
+            VerifyChecksum(LargeText.Decode(streamNoBOM, encodingNoBOM, checksumAlgorigthm, throwIfBinaryDetected: false, canBeEmbedded: false), checksumNoBOM);
+            VerifyChecksum(LargeText.Decode(streamNoBOM, encodingBOM, checksumAlgorigthm, throwIfBinaryDetected: false, canBeEmbedded: false), checksumNoBOM);
 
             // LargeText from stream with BOM. Checksum should include BOM.
-            VerifyChecksum(LargeText.Decode(streamBOM, encodingNoBOM, checksumAlgorigthm, throwIfBinaryDetected: false), checksumBOM);
-            VerifyChecksum(LargeText.Decode(streamBOM, encodingBOM, checksumAlgorigthm, throwIfBinaryDetected: false), checksumBOM);
+            VerifyChecksum(LargeText.Decode(streamBOM, encodingNoBOM, checksumAlgorigthm, throwIfBinaryDetected: false, canBeEmbedded: false), checksumBOM);
+            VerifyChecksum(LargeText.Decode(streamBOM, encodingBOM, checksumAlgorigthm, throwIfBinaryDetected: false, canBeEmbedded: false), checksumBOM);
 
             // LargeText from writer no BOM. Checksum includes BOM
             // from explicit encoding. This is inconsistent with the

src/Compilers/Core/CodeAnalysisTest/Text/TextChangeTests.cs

@@ -558,7 +558,7 @@ public void TestLargeTextWriterReusesLargeChunks()
 
         private SourceText CreateLargeText(params char[][] chunks)
         {
-            return new LargeText(ImmutableArray.Create(chunks), Encoding.UTF8, default(ImmutableArray<byte>), SourceHashAlgorithm.Sha256);
+            return new LargeText(ImmutableArray.Create(chunks), Encoding.UTF8, default(ImmutableArray<byte>), SourceHashAlgorithm.Sha256, default(ImmutableArray<byte>));
         }
 
         private ImmutableArray<char[]> GetChunks(SourceText text)

src/Compilers/Core/Portable/CodeAnalysis.csproj

@@ -52,7 +52,7 @@
   <ItemGroup>
     <Compile Include="CodeGen\ItemTokenMap.cs" />
     <Compile Include="EmbeddedText.cs" />
-    <Compile Include="InternalUtilities\LargeBlobBuildingStream.cs" />
+    <Compile Include="InternalUtilities\BlobBuildingStream.cs" />
     <Compile Include="GlobalSuppressions.cs" />
     <Compile Include="InternalUtilities\IncrementalHash.cs" />
     <Compile Include="PEWriter\DebugSourceInfo.cs" />

src/Compilers/Core/Portable/CodeAnalysisResources.resx

@@ -520,12 +520,15 @@
     <value>Tuples must have at least two elements.</value>
   </data>
   <data name="CompilationReferencesAssembliesWithDifferentAutoGeneratedVersion" xml:space="preserve">
-     <value>The compilation references multiple assemblies whose versions only differ in auto-generated build and/or revision numbers.</value>
+    <value>The compilation references multiple assemblies whose versions only differ in auto-generated build and/or revision numbers.</value>
   </data>
   <data name="TupleUnderlyingTypeMustBeTupleCompatible" xml:space="preserve">
     <value>The underlying type for a tuple must be tuple-compatible.</value>
   </data>
-  <data name="EmbeddedTextMustHaveEncoding" xml:space="preserve">
-    <value>SourceText must have non-null encoding to be embedded.</value>
+  <data name="SourceTextCannotBeEmbedded" xml:space="preserve">
+    <value>SourceText must have been created with canBeEmbedded=true to be embedded.</value>
   </data>
-</root>
+  <data name="TextIsToLongToEmbed" xml:space="preserve">
+    <value>Text is too long to embed.</value>
+  </data>
+</root>

src/Compilers/Core/Portable/CommandLine/CommonCompiler.cs

@@ -137,6 +137,36 @@ internal SourceText ReadFileContent(CommandLineSourceFile file, IList<Diagnostic
             return ReadFileContent(file, diagnostics, out discarded);
         }
 
+        internal EmbeddedText ReadEmbeddedFileContent(CommandLineSourceFile file, IList<DiagnosticInfo> diagnostics)
+        {
+            try
+            {
+                // PERF: Using a very small buffer size for the FileStream opens up an optimization within EncodedStringText where
+                // we read the entire FileStream into a byte array in one shot. For files that are actually smaller than the buffer
+                // size, FileStream.Read still allocates the internal buffer.
+                using (var stream = PortableShim.FileStream.Create(file.Path, PortableShim.FileMode.Open, PortableShim.FileAccess.Read, PortableShim.FileShare.ReadWrite, bufferSize: 1, options: PortableShim.FileOptions.None))
+                {
+                    const int LargeObjectHeapLimit = 80 * 1024;
+
+                    if (stream.Length < LargeObjectHeapLimit)
+                    {
+                        byte[] buffer = EncodedStringText.TryGetByteArrayFromStream(stream);
+                        if (buffer != null)
+                        {
+                            return EmbeddedText.FromBytes(file.Path, buffer, (int)stream.Length, Arguments.ChecksumAlgorithm);
+                        }
+                    }
+
+                    return EmbeddedText.FromStream(file.Path, stream, Arguments.ChecksumAlgorithm);
+                }
+            }
+            catch (Exception e)
+            {
+                diagnostics.Add(ToFileReadDiagnostics(this.MessageProvider, e, file.Path));
+                return null;
+            }
+        }
+
         /// <summary>
         /// Reads content of a source file.
         /// </summary>
@@ -650,17 +680,20 @@ private ImmutableArray<EmbeddedText> AcquireEmbeddedTexts(Compilation compilatio
             foreach (var file in embeddedFiles)
             {
                 SyntaxTree tree;
+                EmbeddedText text;
+
                 if (trees.TryGetValue(file.Path, out tree))
                 {
-                    builder.Add(new EmbeddedText(file.Path, tree.GetText()));
+                    text = EmbeddedText.FromSource(file.Path, tree.GetText());
                 }
                 else
                 {
-                    SourceText text = ReadFileContent(file, diagnostics);
-                    if (text != null)
-                    {
-                        builder.Add(new EmbeddedText(file.Path, text));
-                    }
+                    text = ReadEmbeddedFileContent(file, diagnostics);
+                }
+
+                if (text != null)
+                {
+                    builder.Add(text);
                 }
             }