Build benchmarks in CI

Directory.Build.props

@@ -16,6 +16,10 @@
       <BUILDING_USING_DOTNET>true</BUILDING_USING_DOTNET>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(SolutionName)' == 'FSharp.Benchmarks' and '$(BUILDING_USING_DOTNET)' != 'false'">
+      <BUILDING_USING_DOTNET>true</BUILDING_USING_DOTNET>
+  </PropertyGroup>
+
   <!-- Temporary workaround if proj-info is used, allows Ionide's evaluation to work correctly. -->
   <PropertyGroup Condition="'$(BUILDING_USING_DOTNET)' != 'false' and '$(DotnetProjInfo)' == 'true'">
       <BUILDING_USING_DOTNET>true</BUILDING_USING_DOTNET>

FSharp.Benchmarks.sln

@@ -24,8 +24,6 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Compiler.Benchmarks"
 EndProject
 Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FCSSourceFiles", "tests\benchmarks\FCSBenchmarks\FCSSourceFiles\FCSSourceFiles.fsproj", "{0E2A7B27-3AD3-4C1D-BA0D-008A1200946F}"
 EndProject
-Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Fsharp.ProfilingStartpointProject", "tests\benchmarks\Fsharp.ProfilingStartpointProject\Fsharp.ProfilingStartpointProject.fsproj", "{9F27346B-2FC6-4FD5-A932-4E80F331E6D6}"
-EndProject
 Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Test.Utilities", "tests\FSharp.Test.Utilities\FSharp.Test.Utilities.fsproj", "{0B149238-0912-493E-8877-F831AE01B942}"
 EndProject
 Global
@@ -105,12 +103,6 @@ Global
 		{0E2A7B27-3AD3-4C1D-BA0D-008A1200946F}.Release|Any CPU.Build.0 = Release|Any CPU
 		{0E2A7B27-3AD3-4C1D-BA0D-008A1200946F}.ReleaseCompressed|Any CPU.ActiveCfg = Debug|Any CPU
 		{0E2A7B27-3AD3-4C1D-BA0D-008A1200946F}.Proto|Any CPU.ActiveCfg = Debug|Any CPU
-		{9F27346B-2FC6-4FD5-A932-4E80F331E6D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{9F27346B-2FC6-4FD5-A932-4E80F331E6D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{9F27346B-2FC6-4FD5-A932-4E80F331E6D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{9F27346B-2FC6-4FD5-A932-4E80F331E6D6}.Release|Any CPU.Build.0 = Release|Any CPU
-		{9F27346B-2FC6-4FD5-A932-4E80F331E6D6}.ReleaseCompressed|Any CPU.ActiveCfg = Debug|Any CPU
-		{9F27346B-2FC6-4FD5-A932-4E80F331E6D6}.Proto|Any CPU.ActiveCfg = Debug|Any CPU
 		{0B149238-0912-493E-8877-F831AE01B942}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{0B149238-0912-493E-8877-F831AE01B942}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0B149238-0912-493E-8877-F831AE01B942}.Release|Any CPU.ActiveCfg = Release|Any CPU

VisualFSharp.sln

@@ -189,8 +189,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "FCSBenchmarks", "FCSBenchma
 		tests\benchmarks\FCSBenchmarks\SmokeTestAllBenchmarks.ps1 = tests\benchmarks\FCSBenchmarks\SmokeTestAllBenchmarks.ps1
 	EndProjectSection
 EndProject
-Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Fsharp.ProfilingStartpointProject", "tests\benchmarks\Fsharp.ProfilingStartpointProject\Fsharp.ProfilingStartpointProject.fsproj", "{FE23BB65-276A-4E41-8CC7-F7752241DEBA}"
-EndProject
 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Editor.Tests", "vsintegration\tests\FSharp.Editor.Tests\FSharp.Editor.Tests.fsproj", "{CBC96CC7-65AB-46EA-A82E-F6A788DABF80}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FSharp.Editor.IntegrationTests", "vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj", "{E31F9B59-FCF1-4D04-8762-C7BB60285A7B}"
@@ -997,18 +995,6 @@ Global
 		{583182E1-3484-4A8F-AC06-7C0D232C0CA4}.Release|Any CPU.Build.0 = Release|Any CPU
 		{583182E1-3484-4A8F-AC06-7C0D232C0CA4}.Release|x86.ActiveCfg = Release|Any CPU
 		{583182E1-3484-4A8F-AC06-7C0D232C0CA4}.Release|x86.Build.0 = Release|Any CPU
-		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Debug|x86.Build.0 = Debug|Any CPU
-		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Proto|Any CPU.ActiveCfg = Debug|Any CPU
-		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Proto|Any CPU.Build.0 = Debug|Any CPU
-		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Proto|x86.ActiveCfg = Debug|Any CPU
-		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Proto|x86.Build.0 = Debug|Any CPU
-		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Release|Any CPU.Build.0 = Release|Any CPU
-		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Release|x86.ActiveCfg = Release|Any CPU
-		{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Release|x86.Build.0 = Release|Any CPU
 		{CBC96CC7-65AB-46EA-A82E-F6A788DABF80}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{CBC96CC7-65AB-46EA-A82E-F6A788DABF80}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{CBC96CC7-65AB-46EA-A82E-F6A788DABF80}.Debug|x86.ActiveCfg = Debug|Any CPU
@@ -1111,7 +1097,6 @@ Global
 		{EB015235-1E07-4CDA-9CC6-3FBCC27910D1} = {B8DDA694-7939-42E3-95E5-265C2217C142}
 		{583182E1-3484-4A8F-AC06-7C0D232C0CA4} = {39CDF34B-FB23-49AE-AB27-0975DA379BB5}
 		{39CDF34B-FB23-49AE-AB27-0975DA379BB5} = {DFB6ADD7-3149-43D9-AFA0-FC4A818B472B}
-		{FE23BB65-276A-4E41-8CC7-F7752241DEBA} = {39CDF34B-FB23-49AE-AB27-0975DA379BB5}
 		{CBC96CC7-65AB-46EA-A82E-F6A788DABF80} = {F7876C9B-FB6A-4EFB-B058-D6967DB75FB2}
 		{E31F9B59-FCF1-4D04-8762-C7BB60285A7B} = {F7876C9B-FB6A-4EFB-B058-D6967DB75FB2}
 	EndGlobalSection

azure-pipelines.yml

@@ -713,6 +713,33 @@ stages:
             continueOnError: true
             condition: always()
 
+        # Build benchmarks
+        - job: Plain_Build_Benchmarks
+          pool:
+            name: $(DncEngPublicBuildPool)
+            demands: ImageOverride -equals $(WindowsMachineQueueName)
+          variables:
+          - name: _BuildConfig
+            value: Debug
+          steps:
+          - checkout: self
+            clean: true
+          - script: dotnet --list-sdks
+            displayName: Report dotnet SDK versions
+          - task: UseDotNet@2
+            displayName: install SDK
+            inputs:
+              packageType: sdk
+              useGlobalJson: true
+              includePreviewVersions: true
+              workingDirectory: $(Build.SourcesDirectory)
+              installationPath: $(Agent.ToolsDirectory)/dotnet
+          - script: dotnet build .\FSharp.Benchmarks.sln /bl:\"artifacts/log/$(_BuildConfig)/BenchmarkBuild.binlog\"
+            workingDirectory: $(Build.SourcesDirectory)
+            displayName: Regular rebuild of FSharp.Benchmarks.sln
+            continueOnError: true
+            condition: always()
+
         # Test trimming on Windows
         - job: Build_And_Test_Trimming_Windows
           pool:

buildtools/AssemblyCheck/SkipVerifyEmbeddedPdb.txt

@@ -1,6 +1,5 @@
 FSharp.Build.UnitTests.dll
 FSharp.Compiler.Benchmarks.dll
-Fsharp.ProfilingStartpointProject.dll
 FSharp.Compiler.ComponentTests.dll
 FSharp.Test.Utilities.dll
 FSharp.Compiler.Private.Scripting.UnitTests.dll

tests/benchmarks/CompiledCodeBenchmarks/MicroPerf/CS/MicroPerfCSharp.csproj

@@ -6,11 +6,10 @@
     <LangVersion>8.0</LangVersion>
   </PropertyGroup>
 
-  <ItemGroup>
-    <Compile Remove="obj\**" />
-    <EmbeddedResource Remove="obj\**" />
-    <None Remove="obj\**" />
-  </ItemGroup>
+  <PropertyGroup>
+    <!--missing XML comments-->
+    <NoWarn>$(NoWarn);CS1591</NoWarn>
+  </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="BenchmarkDotNet" Version="0.12.1" />

tests/benchmarks/CompiledCodeBenchmarks/TaskPerf/TaskPerfCSharp/TaskPerfCSharp.csproj

@@ -6,6 +6,11 @@
     <LangVersion>8.0</LangVersion>
   </PropertyGroup>
 
+  <PropertyGroup>
+    <!--missing XML comments-->
+    <NoWarn>$(NoWarn);CS1591</NoWarn>
+  </PropertyGroup>
+
   <ItemGroup>
     <PackageReference Include="BenchmarkDotNet" Version="$(BenchmarkDotNetVersion)" />
     <PackageReference Include="System.Interactive.Async" Version="4.1.1" />

tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/DecentlySizedStandAloneFileBenchmark.fs

@@ -1,22 +1,61 @@
 namespace FSharp.Compiler.Benchmarks
 
 open System.IO
-open HistoricalBenchmark
+open FSharp.Compiler.CodeAnalysis
+open FSharp.Compiler.Text
 open BenchmarkDotNet.Attributes
 
-type SingleFileCompilerWithILCacheClearing(file, options) =
-    inherit SingleFileCompiler(file, options)
-
-    override this.Cleanup() =
-        base.Cleanup()
-        FSharp.Compiler.AbstractIL.ILBinaryReader.ClearAllILModuleReaderCache()
+type private SingleFileCompilerConfig =
+    {
+        Checker : FSharpChecker
+        Options : FSharpProjectOptions
+    }
 
 [<MemoryDiagnoser>]
 type DecentlySizedStandAloneFileBenchmark() =
-    inherit SingleFileCompilerBenchmarkBase(
-        SingleFileCompilerWithILCacheClearing(
-            Path.Combine(__SOURCE_DIRECTORY__, "../decentlySizedStandAloneFile.fs"),
-            OptionsCreationMethod.FromScript
-        )
-    )
 
+    let mutable configOpt : SingleFileCompilerConfig option = None
+    let filePath = Path.Combine(__SOURCE_DIRECTORY__, "../decentlySizedStandAloneFile.fs")
+
+    let getFileSourceText (filePath : string) =
+        let text = File.ReadAllText(filePath)
+        SourceText.ofString text
+
+    let getConfig () =
+        configOpt
+        |> Option.defaultWith (fun () -> failwith "Setup not run")
+
+    [<GlobalSetup>]
+    member _.Setup() =
+        configOpt <-
+            match configOpt with
+            | Some _ -> configOpt
+            | None ->
+                let checker = FSharpChecker.Create(projectCacheSize = 200)
+                let options =
+                    checker.GetProjectOptionsFromScript(filePath, getFileSourceText filePath)
+                    |> Async.RunSynchronously
+                    |> fst
+                {
+                    Checker = checker
+                    Options = options
+                }
+                |> Some
+
+    [<Benchmark>]
+    member _.Run() =
+        let config = getConfig()
+        let _, result =
+            config.Checker.ParseAndCheckFileInProject(filePath, 0, getFileSourceText filePath, config.Options)
+            |> Async.RunSynchronously
+
+        match result with
+        | FSharpCheckFileAnswer.Aborted -> failwith "checker aborted"
+        | FSharpCheckFileAnswer.Succeeded results ->
+            if results.Diagnostics.Length > 0 then failwithf $"had errors: %A{results.Diagnostics}"
+
+    [<IterationCleanup>]
+    member _.Cleanup() =
+        let checker = getConfig().Checker
+        checker.InvalidateAll()
+        checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients()

tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FSharp.Compiler.Benchmarks.fsproj

@@ -24,7 +24,6 @@
   <ItemGroup>
     <ProjectReference Include="..\..\..\..\src\Compiler\FSharp.Compiler.Service.fsproj" />
     <ProjectReference Include="..\..\..\FSharp.Test.Utilities\FSharp.Test.Utilities.fsproj" />
-    <ProjectReference Include="..\BenchmarkComparison\HistoricalBenchmark.fsproj" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(FSHARPCORE_USE_PACKAGE)' != 'true'">