Add the ability to assert about the nature of a failing test's failur…

…e so that breakages aren't hidden
D-R-R · Mar 23, 2013 · c9ea527 · c9ea527
1 parent ecdac93
commit c9ea527
Show file tree

Hide file tree

Showing 7 changed files with 131 additions and 41 deletions.
diff --git a/Tests/ComparisonTest.cs b/Tests/ComparisonTest.cs
@@ -223,7 +223,8 @@ public static Configuration MakeDefaultConfiguration () {
 
         public TOutput Translate<TOutput> (
             Func<TranslationResult, TOutput> processResult,
-            Func<Configuration> makeConfiguration = null
+            Func<Configuration> makeConfiguration = null,
+            Action<Exception> onTranslationFailure = null
         ) {
             Configuration configuration;
 
@@ -239,26 +240,35 @@ public TOutput Translate<TOutput> (
 
             using (var translator = new JSIL.AssemblyTranslator(configuration, TypeInfo, null, AssemblyCache)) {
                 var assemblyPath = Util.GetPathOfAssembly(Assembly);
+                TranslationResult translationResult = null;
 
-                var translationResult = translator.Translate(
-                    assemblyPath, TypeInfo == null
-                );
+                try {
+                    translationResult = translator.Translate(
+                        assemblyPath, TypeInfo == null
+                    );
 
-                AssemblyTranslator.GenerateManifest(translator.Manifest, assemblyPath, translationResult);
+                    AssemblyTranslator.GenerateManifest(translator.Manifest, assemblyPath, translationResult);
 
-                result = processResult(translationResult);
+                    result = processResult(translationResult);
+                } finally {
+                    if (onTranslationFailure != null) {
+                        foreach (var failure in translator.Failures)
+                            onTranslationFailure(failure);
+                    }
 
-                // If we're using a preconstructed type information provider, we need to remove the type information
-                //  from the assembly we just translated
-                if (TypeInfo != null) {
-                    Assert.AreEqual(1, translationResult.Assemblies.Count);
-                    TypeInfo.Remove(translationResult.Assemblies.ToArray());
-                }
+                    // If we're using a preconstructed type information provider, we need to remove the type information
+                    //  from the assembly we just translated
+                    if ((TypeInfo != null) && (translationResult != null)) {
+                        Assert.AreEqual(1, translationResult.Assemblies.Count);
+                        TypeInfo.Remove(translationResult.Assemblies.ToArray());
+                    }
+
+                    // If we're using a preconstructed assembly cache, make sure the test case assembly didn't get into
+                    //  the cache, since that would leak memory.
+                    if (AssemblyCache != null) {
+                        AssemblyCache.TryRemove(Assembly.FullName);
+                    }
 
-                // If we're using a preconstructed assembly cache, make sure the test case assembly didn't get into
-                //  the cache, since that would leak memory.
-                if (AssemblyCache != null) {
-                    AssemblyCache.TryRemove(Assembly.FullName);
                 }
             }
 
@@ -267,13 +277,13 @@ public TOutput Translate<TOutput> (
 
         public string GenerateJavascript (
             string[] args, out string generatedJavascript, out long elapsedTranslation,
-            Func<Configuration> makeConfiguration = null
+            Func<Configuration> makeConfiguration = null,
+            Action<Exception> onTranslationFailure = null
         ) {
-
             var translationStarted = DateTime.UtcNow.Ticks;
 
             string translatedJs = Translate(
-                (tr) => tr.WriteToString(), makeConfiguration
+                (tr) => tr.WriteToString(), makeConfiguration, onTranslationFailure
             );
 
             elapsedTranslation = DateTime.UtcNow.Ticks - translationStarted;
@@ -330,28 +340,31 @@ public string GenerateJavascript (
         }
 
         public string RunJavascript (
-            string[] args, Func<Configuration> makeConfiguration = null
+            string[] args, Func<Configuration> makeConfiguration = null,
+            Action<Exception> onTranslationFailure = null
         ) {
             string temp1, temp4, temp5;
             long temp2, temp3;
 
-            return RunJavascript(args, out temp1, out temp2, out temp3, out temp4, out temp5, makeConfiguration);
+            return RunJavascript(args, out temp1, out temp2, out temp3, out temp4, out temp5, makeConfiguration, onTranslationFailure);
         }
 
         public string RunJavascript (
             string[] args, out string generatedJavascript, out long elapsedTranslation, out long elapsedJs,
-            Func<Configuration> makeConfiguration = null
+            Func<Configuration> makeConfiguration = null,
+            Action<Exception> onTranslationFailure = null
         ) {
             string temp1, temp2;
 
-            return RunJavascript(args, out generatedJavascript, out elapsedTranslation, out elapsedJs, out temp1, out temp2, makeConfiguration);
+            return RunJavascript(args, out generatedJavascript, out elapsedTranslation, out elapsedJs, out temp1, out temp2, makeConfiguration, onTranslationFailure);
         }
 
         public string RunJavascript (
             string[] args, out string generatedJavascript, out long elapsedTranslation, out long elapsedJs, out string stderr, out string trailingOutput,
-            Func<Configuration> makeConfiguration = null
+            Func<Configuration> makeConfiguration = null,
+            Action<Exception> onTranslationFailure = null
         ) {
-            var tempFilename = GenerateJavascript(args, out generatedJavascript, out elapsedTranslation, makeConfiguration);
+            var tempFilename = GenerateJavascript(args, out generatedJavascript, out elapsedTranslation, makeConfiguration, onTranslationFailure);
 
             using (var evaluator = EvaluatorPool.Get()) {
                 var startedJs = DateTime.UtcNow.Ticks;
@@ -436,7 +449,12 @@ public string RunJavascript (
             }
         }
 
-        public void Run (string[] args = null, Func<Configuration> makeConfiguration = null, bool dumpJsOnFailure = true) {
+        public void Run (
+            string[] args = null, 
+            Func<Configuration> makeConfiguration = null, 
+            bool dumpJsOnFailure = true,
+            Action<Exception> onTranslationFailure = null
+        ) {
             var signals = new[] {
                     new ManualResetEventSlim(false), new ManualResetEventSlim(false)
                 };
@@ -462,7 +480,11 @@ public void Run (string[] args = null, Func<Configuration> makeConfiguration = n
 
             ThreadPool.QueueUserWorkItem((_) => {
                 try {
-                    outputs[1] = RunJavascript(args, out generatedJs[0], out elapsed[1], out elapsed[2], makeConfiguration: makeConfiguration).Replace("\r", "").Trim();
+                    outputs[1] = RunJavascript(
+                        args, out generatedJs[0], out elapsed[1], out elapsed[2], 
+                        makeConfiguration: makeConfiguration,
+                        onTranslationFailure: onTranslationFailure
+                    ).Replace("\r", "").Trim();
                 } catch (Exception ex) {
                     errors[1] = ex;
                 }

diff --git a/Tests/FailingTestCases/PassNormalArrayAsPackedArrayArgument.cs b/Tests/FailingTestCases/PassNormalArrayAsPackedArrayArgument.cs
@@ -1,4 +1,7 @@
-using System;
+//@assertFailureString Invalid attempt to pass a normal array as parameter
+//@assertThrows JavaScriptEvaluatorException
+
+using System;
 using JSIL.Meta;
 
 public struct IntFloatPair {

diff --git a/Tests/FailingTestCases/PassPackedArrayAsNormalArrayArgument.cs b/Tests/FailingTestCases/PassPackedArrayAsNormalArrayArgument.cs
@@ -1,4 +1,7 @@
-using System;
+//@assertFailureString Invalid attempt to pass a packed array as parameter
+//@assertThrows JavaScriptEvaluatorException
+
+using System;
 using JSIL.Meta;
 
 public struct IntFloatPair {

diff --git a/Tests/FailingTestCases/ReturnNormalArrayAsPackedArray.cs b/Tests/FailingTestCases/ReturnNormalArrayAsPackedArray.cs
@@ -1,4 +1,7 @@
-using System;
+//@assertFailureString Return value of method 'ReturnPackedArray' must be a packed array
+//@assertThrows JavaScriptEvaluatorException
+
+using System;
 using JSIL.Meta;
 
 public struct IntFloatPair {

diff --git a/Tests/FailingTestCases/ReturnPackedArrayAsNormalArray.cs b/Tests/FailingTestCases/ReturnPackedArrayAsNormalArray.cs
@@ -1,4 +1,7 @@
-using System;
+//@assertFailureString Return value of method 'ReturnPackedArray' is a packed array
+//@assertThrows JavaScriptEvaluatorException
+
+using System;
 using JSIL.Meta;
 
 public struct IntFloatPair {

diff --git a/Tests/FailingTests.cs b/Tests/FailingTests.cs
@@ -6,26 +6,77 @@
 using System.Reflection;
 using System.Runtime.Serialization.Json;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Threading;
 using JSIL.Internal;
 using NUnit.Framework;
 
 namespace JSIL.Tests {
     [TestFixture]
     public class FailingTests : GenericTestFixture {
+        public static Regex MetacommentRegex = new Regex(
+            @"//@(?'command'[A-Za-z_0-9]+) (?'arguments'[^\n\r]*)",
+            RegexOptions.ExplicitCapture
+        );
+
         [Test]
         [TestCaseSource("FailingTestCasesSource")]
         public void FailingTestCases (object[] parameters) {
             var passed = false;
 
+            var testFilename = (string)parameters[0];
+            var translationFailures = new List<Exception>();
+            Exception thrown = null;
+
             try {
-                RunSingleComparisonTestCase(parameters);
+                RunSingleComparisonTestCase(
+                    parameters, 
+                    makeConfiguration: () => {
+                        var cfg = MakeConfiguration();
+                        cfg.UseThreads = false;
+                        return cfg;
+                    },
+                    onTranslationFailure: (exc) => {
+                        lock (translationFailures)
+                            translationFailures.Add(exc);
+                    }
+                );
 
                 passed = true;
-            } catch (JavaScriptEvaluatorException jse) {
-                Console.WriteLine(jse.ToString());
-            } catch (AssertionException ex) {
-                Console.WriteLine(ex.ToString());
+            } catch (Exception exc) {
+                thrown = exc;
+            }
+
+            foreach (var failure in translationFailures)
+                Console.WriteLine(failure.ToString());
+
+            var testFileText = File.ReadAllText(testFilename);
+            foreach (Match metacommentMatch in MetacommentRegex.Matches(testFileText)) {
+                Console.WriteLine(metacommentMatch.Value);
+
+                var command = metacommentMatch.Groups["command"].Value.ToLower();
+                var args = metacommentMatch.Groups["arguments"].Value;
+
+                switch (command) {
+                    case "assertfailurestring":
+                        Assert.IsTrue(
+                            translationFailures.Any(
+                                (f) => f.ToString().Contains(args)
+                            ),
+                            "Expected translation to generate a failure containing the string '" + args + "'"
+                        );
+
+                        break;
+
+                    case "assertthrows":
+                        if ((thrown == null) || (thrown.GetType().Name.ToLower() != args.Trim().ToLower()))
+                            Assert.Fail("Expected test to throw an exception of type '" + args + "'");
+
+                        break;
+
+                    default:
+                        throw new NotImplementedException("Command type '" + command + "' not supported in metacomments");
+                }
             }
 
             Assert.IsFalse(passed, "Test passed when it should have failed");

diff --git a/Tests/GenericTestFixture.cs b/Tests/GenericTestFixture.cs
@@ -205,7 +205,7 @@ protected void RunComparisonTests (
         private void RunComparisonTest (
             string filename, string[] stubbedAssemblies = null, TypeInfoProvider typeInfo = null, Action<string, string> errorCheckPredicate = null,
             List<string> failureList = null, string commonFile = null, bool shouldRunJs = true, AssemblyCache asmCache = null,
-            Func<Configuration> makeConfiguration = null
+            Func<Configuration> makeConfiguration = null, Action<Exception> onTranslationFailure = null 
         ) {
             Console.WriteLine("// {0} ... ", Path.GetFileName(filename));
 
@@ -224,13 +224,13 @@ private void RunComparisonTest (
                     stubbedAssemblies, typeInfo, asmCache)
                     ) {
                     if (shouldRunJs) {
-                        test.Run(makeConfiguration: makeConfiguration);
+                        test.Run(makeConfiguration: makeConfiguration, onTranslationFailure: onTranslationFailure);
                     } else {
                         string js;
                         long elapsed;
                         try {
                             var csOutput = test.RunCSharp(new string[0], out elapsed);
-                            test.GenerateJavascript(new string[0], out js, out elapsed, makeConfiguration);
+                            test.GenerateJavascript(new string[0], out js, out elapsed, makeConfiguration, onTranslationFailure);
 
                             Console.WriteLine("generated");
 
@@ -341,7 +341,11 @@ protected string GenericIgnoreTest (string fileName, string workingOutput, strin
             return generatedJs;
         }
 
-        protected void RunSingleComparisonTestCase (object[] parameters, Func<Configuration> makeConfiguration = null) {
+        protected void RunSingleComparisonTestCase (
+            object[] parameters, 
+            Func<Configuration> makeConfiguration = null,
+            Action<Exception> onTranslationFailure = null
+        ) {
             if (parameters.Length != 5)
                 throw new ArgumentException("Wrong number of test case data parameters.");
 
@@ -350,7 +354,8 @@ protected void RunSingleComparisonTestCase (object[] parameters, Func<Configurat
             try {
                 RunComparisonTest(
                     (string)parameters[0], null, provider, null, null, (string)parameters[3], true, cache,
-                    makeConfiguration: makeConfiguration
+                    makeConfiguration: makeConfiguration,
+                    onTranslationFailure: onTranslationFailure
                 );
             } finally {
                 if ((bool)parameters[4]) {