[CI] Fixing the build

- Also updated build versioing scheme so build and patch are now based on number of tags this month and git height (commits since last tag) respectively. - fix bad paths in scripts - Fix F# string to matrix converter - new lines in string constants match file encoding. - Reduced logging output for tests - Added new log level for prompts (ensures prompts are alwyas rendered) - Fixed tests that are sensitive to small deltas by adding a threshold - The FrequencyScale tests failed because the line endings of the results were different on the CI server. I've adjusted the assertion to be robust to these differences.
QutEcoacoustics · Mar 2, 2018 · 684a836 · 684a836
1 parent ffab631
commit 684a836
Show file tree

Hide file tree

Showing 18 changed files with 119 additions and 76 deletions.
diff --git a/.gitignore b/.gitignore
@@ -25,6 +25,10 @@ $RECYCLE.BIN/
 # Roslyn files
 *.sln.ide
 
+# Cake
+
+/tools
+
 # Build results
 [Dd]ebug/
 [Dd]ebugPublic/
@@ -125,16 +129,6 @@ _TeamCity*
 _NCrunch_*
 .*crunch*.local.xml
 
-# MightyMoose
-*.mm.*
-AutoTest.Net/
-
-# Web workbench (sass)
-.sass-cache/
-
-# Installshield output folder
-[Ee]xpress/
-
 # DocProject is a documentation generator add-in
 DocProject/buildhelp/
 DocProject/Help/*.HxT
@@ -157,12 +151,7 @@ publish/
 # NuGet Packages Directory
 packages/
 .nuget/
-## TODO: If the tool you use requires repositories.config uncomment the next line
-#!packages/repositories.config
 
-# Enable "build/" folder in the NuGet Packages folder since NuGet packages use it for MSBuild targets
-# This line needs to be after the ignore of the build folder (and the packages folder if the line above has been uncommented)
-!packages/build/
 
 # Windows Azure Build Output
 csx/
@@ -193,23 +182,6 @@ Backup*/
 UpgradeLog*.XML
 UpgradeLog*.htm
 
-# SQL Server files
-*.mdf
-*.ldf
-
-# Business Intelligence projects
-*.rdl.data
-*.bim.layout
-*.bim_*.settings
-
-# Microsoft Fakes
-FakesAssemblies/
-
-
-#ncrunch
-*ncrunch*
-*crunch*.local.xml
-
 # matlab
 *.asv
 *.Rhistory

diff --git a/appveyor.yml b/appveyor.yml
@@ -8,17 +8,16 @@ skip_commits:
   message: '/\[chore\]|\[no_ci\]|\[ci_skip\]|\[skip_ci\]/'
   # Do not trigger a build for anything not .NET
   files:
-    - 'AudioAnalysis/Matlab/**/*'
-    - 'AudioAnalysis/RCode/**/*'
     - '**/*.md'
-    - 'doc/**/*'
+    - 'docs/**/*'
+    - 'scripts/**/*'
 
 # Do not run a build on new tags - we use tags to create releases, which can
-# only occur after sucessful build on another branch
+# only occur after successful build on another branch
 skip_tags: true
 
 cache:
-  - AudioAnalysis\packages -> **\packages.config
+  - packages
   # cache the lfs objects directory so that git-lfs does not always download a
   # full set of blobs (saves bandwidth)
   - '.git\lfs\objects'
@@ -29,9 +28,7 @@ matrix:
 platform:
   - Any CPU
 
-
 # Attempt to limit the number of commits to clone - our project has a LOT of history
-# TODO: drop this to about ~100 later on
 clone_depth: 200
 
 environment:
@@ -56,22 +53,21 @@ before_build:
   - ps: $env:RELEASE_COMMIT = $env:APPVEYOR_REPO_COMMIT_MESSAGE_EXTENDED -ilike "*\[release\]*"; $env:SHOULD_RELEASE = ($env:RELEASE_COMMIT -ieq "true") -or ($env:APPVEYOR_SCHEDULED_BUILD -and ($env:TAG_NOT_PREVIOUSLY_RELEASED -ieq "true"));
   - ps: ls env:* -Include SHOULD*,APPVEYOR_SCHEDULED_BUILD*,APPVEYOR_REPO_COMMIT_MESSAGE_EXTENDED*,RELEASE_COMMIT,TAG_NOT_PREVIOUSLY_RELEASED
   # Restore packages 
-  - cd AudioAnalysis
   - nuget restore
 
 build_script:
   - echo "Building Debug"
   - ps: >
       msbuild 
-      "C:\projects\audio-analysis\AudioAnalysis\AudioAnalysis2012.sln" /m /verbosity:minimal 
+      "C:\projects\audio-analysis\AudioAnalysis.sln" /m /verbosity:minimal 
       /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll"
       /p:WarningLevel=0 /p:RunCodeAnalysis=false
       /p:Configuration=Debug /property:Platform=$env:platform
 
   - echo "Building Release"
   - ps: >
       msbuild 
-      "C:\projects\audio-analysis\AudioAnalysis\AudioAnalysis2012.sln" /m /verbosity:minimal 
+      "C:\projects\audio-analysis\AudioAnalysis.sln" /m /verbosity:minimal 
       /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll"
       /p:WarningLevel=0 /p:RunCodeAnalysis=false
       /p:Configuration=Release /property:Platform=$env:platform
@@ -80,15 +76,15 @@ after_build:
   # package up artifacts
   # defined env vars: $env:ApPackage{Configuration} , $env:ApVersion , $env:ApName{Configuration}
   - cd %APPVEYOR_BUILD_FOLDER%
-  - ps: . .\package.ps1 "Debug"
-  - ps: . .\package.ps1 "Release"
+  - ps: . .\build\package.ps1 "Debug"
+  - ps: . .\build\package.ps1 "Release"
   #- ps: 'ls env:'
 
 test:
   assemblies:
     only:
-      #- "Acoustics\Acoustics.Test\bin\Debug\Acoustics.Test.dll"
-      - '**\bin\Debug\Acoustics.Test.dll'
+      - 'tests\Acoustics.Test\bin\Debug\Acoustics.Test.dll'
+      - 'tests\AED.Test\bin\Debug\AED.Test.dll'
 
 # Upload previously generated artifacts
 artifacts:

diff --git a/build/package.ps1 b/build/package.ps1
@@ -38,9 +38,8 @@ if ($configuration -eq $null) {
 
 cd $PSScriptRoot
 try {
-Push-Location
 
-cd "AudioAnalysis/AnalysisPrograms/bin"
+cd "$PSScriptRoot/../src/AnalysisPrograms/bin"
 
 
 echo "Removing old zips (this should have no effect on CI server)"
@@ -60,9 +59,9 @@ exec { 7z a -tzip $ApName "./$configuration/*" -xr0!*log.txt* }
 
 echo "Packing complete"
 
-Set-Item "env:ApPackage$Configuration" (Join-Path "AudioAnalysis\AnalysisPrograms\bin" $ApName)
+Set-Item "env:ApPackage$Configuration" (Join-Path "src\AnalysisPrograms\bin" $ApName)
 
 }
 finally {
-Pop-Location
+cd "$PSScriptRoot/.."
 }
diff --git a/build/release.ps1 b/build/release.ps1
@@ -49,7 +49,7 @@ if (!$ci) {
   # create and upload a github release
   echo "creating github release"
 
-  $artifacts = ((ls .\AudioAnalysis\AnalysisPrograms\bin\*.zip) | % { "-a " + $_ }) -join " "
+  $artifacts = ((ls .\src\AnalysisPrograms\bin\*.zip) | % { "-a " + $_ }) -join " "
 
   hub release create "$(if($pre_release){"-p"})" $artifacts -m $release_message $tag_name
 

diff --git a/src/Acoustics.Shared/Logging/LogExtensions.cs b/src/Acoustics.Shared/Logging/LogExtensions.cs
@@ -16,7 +16,26 @@ namespace log4net
     public static class LogExtensions
     {
         // equivalent to NOTICE
-        private static readonly log4net.Core.Level SuccessLevel = new log4net.Core.Level(50000, "SUCCESS");
+        public static readonly Core.Level SuccessLevel = new Core.Level(50000, "SUCCESS");
+
+        // Higher than all other levels, but lower than OFF.
+        // In interactive scenarios we need to be sure that the user sees the message.
+        public static readonly Core.Level PromptLevel = new Core.Level(150_000, "PROMPT");
+
+        /// <summary>
+        /// Log a message object with the <see cref="F:LogExtensions.PromptLevel"/> level.
+        /// Use this mehtod only for interactive prompts that the user must see.
+        /// </summary>
+        /// <param name="log">
+        /// The log.
+        /// </param>
+        /// <param name="message">
+        /// The message.
+        /// </param>
+        public static void Prompt(this ILog log, object message)
+        {
+            log.Logger.Log(null, PromptLevel, message, null);
+        }
 
         /// <summary>
         /// Log a message object with the <see cref="F:LogExtensions.SuccessLevel"/> level -

diff --git a/src/Acoustics.Shared/Logging/LoggedConsole.cs b/src/Acoustics.Shared/Logging/LoggedConsole.cs
@@ -107,7 +107,7 @@ public static void WriteFatalLine(string str)
 
         public static void WriteWaitingLine<T>(Task<T> task, string message = null)
         {
-            WriteLine(message ?? "Waiting...");
+            Log.Prompt(message ?? "Waiting...");
             if (IsInteractive)
             {
                 var spinner = new DotSpinner(task);
@@ -119,15 +119,15 @@ public static string Prompt(string prompt, bool forPassword = false, TimeSpan? t
         {
             if (IsInteractive)
             {
-                WriteLine(prompt);
+                Log.Prompt(prompt);
                 var d = new Func<string>(() =>
                 {
                     if (forPassword)
                     {
                         return ReadHiddenLine();
                     }
 
-                    var line = System.Console.ReadLine();
+                    var line = Console.ReadLine();
                     return line;
                 });
                 IAsyncResult result = d.BeginInvoke(null, null);

diff --git a/src/AnalysisPrograms/AnalysisPrograms.csproj b/src/AnalysisPrograms/AnalysisPrograms.csproj
@@ -467,6 +467,16 @@
     <!--    <GitCommits>-->
     <!--      <Output TaskParameter="CommitsCount" PropertyName="GitCommitsCount" />-->
     <!--    </GitCommits>-->
+    <PropertyGroup>
+      <TagCommand>--tags --simplify-by-decoration --pretty="format:%ai %d" --after="$([System.DateTime]::Now.ToString("yyyy-MM-01T00:00Z"))"</TagCommand>
+    </PropertyGroup>
+    <GitClient
+      Command="log"
+      Arguments="$(TagCommand)">
+      <Output TaskParameter="ConsoleOutput" ItemName="GitTagCount" />
+    </GitClient>
+    <Message Text="Tags command: $(TagCommand)" Importance="High" />
+    <Message Text="Tags this month: @(GitTagCount)" Importance="High" />
     <GitPendingChanges>
       <Output TaskParameter="HasPendingChanges" PropertyName="GitHasPendingChanges" />
     </GitPendingChanges>
@@ -479,9 +489,9 @@
       <month>$([System.DateTime]::Now.ToString("MM"))</month>
       <APPVEYOR_BUILD_NUMBER Condition="''=='$(APPVEYOR_BUILD_NUMBER)'">000</APPVEYOR_BUILD_NUMBER>
       <GitBranch Condition="''=='$(GitBranch)'">$(APPVEYOR_REPO_BRANCH)</GitBranch>
-      <version>$(year).$(month).$(APPVEYOR_BUILD_NUMBER).$(GitDescribeCommitCount)</version>
+      <version>$(year).$(month).@(GitTagCount->Count()).$(GitDescribeCommitCount)</version>
       <isDirty Condition="$(GitHasPendingChanges)">-DIRTY</isDirty>
-      <gitVersion>$(GitBranch)-$(GitCommitHash)$(isDirty)</gitVersion>
+      <gitVersion>$(GitBranch)-$(GitCommitHash)$(isDirty)-CI:$(APPVEYOR_BUILD_NUMBER)</gitVersion>
     </PropertyGroup>
     <ItemGroup>
       <Tokens Include="Version">
@@ -490,6 +500,9 @@
       <Tokens Include="GitVersion">
         <ReplacementValue>$(gitVersion)</ReplacementValue>
       </Tokens>
+      <Tokens Include="Year">
+        <ReplacementValue>20$(year)</ReplacementValue>
+      </Tokens>
     </ItemGroup>
     <Message Text="Last Tag: $(GitDescribeTag)%0aHash: $(GitDescribeHash)%0aCommitCount:$(GitDescribeCommitCount)" Importance="High" />
     <Message Text="Old assembly version: $(oldVersion). Replacing with: $(version)!" Importance="High" />

diff --git a/src/AnalysisPrograms/Production/MainEntryUtilities.cs b/src/AnalysisPrograms/Production/MainEntryUtilities.cs
@@ -406,7 +406,8 @@ public static void SetLogVerbosity(LogVerbosity logVerbosity, bool quietConsole
             switch (logVerbosity)
             {
                 case LogVerbosity.None:
-                    modifiedLevel = Level.Off;
+                    // we never turn the logger completely off - sometimes the logger just really needs to log something.
+                    modifiedLevel = LogExtensions.PromptLevel;
                     break;
                 case LogVerbosity.Error:
                     modifiedLevel = Level.Error;

diff --git a/src/AnalysisPrograms/Properties/AssemblyInfo.cs.template b/src/AnalysisPrograms/Properties/AssemblyInfo.cs.template
@@ -19,7 +19,7 @@ using System.Runtime.InteropServices;
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("QUT")]
 [assembly: AssemblyProduct("AnalysisPrograms")]
-[assembly: AssemblyCopyright("Copyright © QUT 2007-2015")]
+[assembly: AssemblyCopyright("Copyright © QUT 2007-${Year}")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 

diff --git a/tests/AED.Test/Common.fs b/tests/AED.Test/Common.fs
@@ -25,8 +25,9 @@ let matlabPath = @"..\..\..\Fixtures\FSharp\"
 do
     let unzip file =
         let path = Path.Combine(matlabPath, file)
-        Directory.Delete(path, true)
-        ZipFile.ExtractToDirectory(path+ ".zip", matlabPath)
+        if Directory.Exists(path) then
+            Directory.Delete(path, true)
+        ZipFile.ExtractToDirectory(path + ".zip", matlabPath)
     unzip BAC2_20071015_045040.Dir
     unzip GParrots_JB2_20090607_173000_wav_minute_3.Dir
 
@@ -98,7 +99,8 @@ let createEvents (rs:seq<Rectangle<int, int>>) = Seq.map (fun r -> { Bounds = r;
 
 
 let parseStringAsMatrix (input:string) =
-    let split = input.Split([|Environment.NewLine|], StringSplitOptions.RemoveEmptyEntries)
+    let delimitters = [|for  c in Environment.NewLine -> c.ToString()|]
+    let split = input.Split(delimitters, StringSplitOptions.RemoveEmptyEntries)
 
     Matrix.init (split.Length) (split.[0].Length) (fun x y -> split.[x].[y] |> string |> Double.Parse)
 

diff --git a/tests/Acoustics.Test/Acoustics.Test.csproj b/tests/Acoustics.Test/Acoustics.Test.csproj
@@ -267,6 +267,7 @@
     <Compile Include="Shared\ConfigTests.cs" />
     <Compile Include="Shared\RangeTests.cs" />
     <Compile Include="TestHelpers\Factories\AudioRecordingFactory.cs" />
+    <Compile Include="TestHelpers\Setup.cs" />
     <Compile Include="Tools\AudioFilePreparerTests.cs" />
     <Compile Include="FileDateHelpersTests.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />

diff --git a/tests/Acoustics.Test/AnalysisPrograms/AnalyzeLongRecordings/TestAnalyzeLongRecording.cs b/tests/Acoustics.Test/AnalysisPrograms/AnalyzeLongRecordings/TestAnalyzeLongRecording.cs
@@ -5,6 +5,7 @@
 namespace Acoustics.Test.AnalysisPrograms.AnalyzeLongRecordings
 {
     using System;
+    using System.Diagnostics;
     using System.IO;
     using System.Linq;
     using Acoustics.Shared;
@@ -89,6 +90,7 @@ public void TestAnalyzeSr22050Recording()
                 Config = configPath.FullName,
                 Output = this.outputDirectory,
                 MixDownToMono = true,
+                Parallel = !Debugger.IsAttached,
             };
 
             AnalyseLongRecording.Execute(arguments);
@@ -197,6 +199,7 @@ public void TestAnalyzeSr64000Recording()
                 Config = newConfigPath.FullName,
                 Output = this.outputDirectory,
                 MixDownToMono = true,
+                Parallel = !Debugger.IsAttached,
             };
 
             AnalyseLongRecording.Execute(arguments);

diff --git a/tests/Acoustics.Test/AnalysisPrograms/Draw/Zooming/DrawZoomingTests.cs b/tests/Acoustics.Test/AnalysisPrograms/Draw/Zooming/DrawZoomingTests.cs
@@ -46,9 +46,7 @@ public static void ClassInitialize(TestContext context)
             };
 
             context.WriteLine($"{DateTime.Now} generating indices fixture data");
-            MainEntry.SetLogVerbosity(LogVerbosity.Warn, false);
             AnalyseLongRecording.Execute(arguments);
-            MainEntry.SetLogVerbosity(LogVerbosity.Debug, false);
             context.WriteLine($"{DateTime.Now} finished generting fixture");
 
             ResultsDirectory = SharedDirectory.Combine("Towsey.Acoustic");

diff --git a/tests/Acoustics.Test/AudioAnalysisTools/DSP/EnvelopeAndFftTests.cs b/tests/Acoustics.Test/AudioAnalysisTools/DSP/EnvelopeAndFftTests.cs
@@ -29,6 +29,7 @@ namespace Acoustics.Test.AudioAnalysisTools.DSP
     public class EnvelopeAndFftTests
     {
         private DirectoryInfo outputDirectory;
+        public const double Delta = 0.000_000_001;
 
         [TestInitialize]
         public void Setup()
@@ -117,8 +118,8 @@ public void TestEnvelopeAndFft1()
             var expectedColSums = Binary.Deserialize<double[]>(sumFile);
             var totalDelta = expectedColSums.Zip(columnSums, ValueTuple.Create).Select(x => Math.Abs(x.Item1 - x.Item2)).Sum();
             var avgDelta = expectedColSums.Zip(columnSums, ValueTuple.Create).Select(x => Math.Abs(x.Item1 - x.Item2)).Average();
-            Assert.AreEqual(expectedColSums[0], columnSums[0], $"\nE: {expectedColSums[0]:R}\nA: {columnSums[0]:R}\nD: {expectedColSums[0] - columnSums[0]:R}\nT: {totalDelta:R}\nA: {avgDelta}\nn: {expectedColSums.Length}");
-            CollectionAssert.AreEqual(expectedColSums, columnSums);
+            Assert.AreEqual(expectedColSums[0], columnSums[0], Delta, $"\nE: {expectedColSums[0]:R}\nA: {columnSums[0]:R}\nD: {expectedColSums[0] - columnSums[0]:R}\nT: {totalDelta:R}\nA: {avgDelta}\nn: {expectedColSums.Length}");
+            CollectionAssert.That.AreEqual(expectedColSums, columnSums, Delta);
         }
 
         /// <summary>
@@ -212,7 +213,7 @@ public void TestEnvelopeAndFft2()
             //Binary.Serialize(frameDecibelsFile, frameDecibels);
 
             var expectedFrameDecibels = Binary.Deserialize<double[]>(frameDecibelsFile);
-            CollectionAssert.AreEqual(expectedFrameDecibels, frameDecibels);
+            CollectionAssert.That.AreEqual(expectedFrameDecibels, frameDecibels, Delta);
 
             // freq info
             Assert.AreEqual(255, nyquistBin);