Fix 2906

CHANGES.txt

@@ -1,6 +1,7 @@
 Current
-New: GITHUB-2897: Not exception but warning if some (not all) of the given test names are not found in suite files. (Bruce Wen)
-New: Added assertListContains and assertListContainsObject methods to check if specific object present in List (Dmytro Budym)
+Fixed: GITHUB-2906: Generate testng-results.xml per test suite (Krishnan Mahadevan)
+New:   GITHUB-2897: Not exception but warning if some (not all) of the given test names are not found in suite files. (Bruce Wen)
+New:   GITHUB-2907: Added assertListContains and assertListContainsObject methods to check if specific object present in List (Dmytro Budym)
 Fixed: GITHUB-2888: Skipped Tests with DataProvider appear as failed (Joaquin Moreira)
 Fixed: GITHUB-2884: Discrepancies with DataProvider and Retry of failed tests (Krishnan Mahadevan)
 Fixed: GITHUB-2879: Test listeners specified in parent testng.xml file are not included in testng-failed.xml file (Krishnan Mahadevan)

testng-core/src/main/java/org/testng/CommandLineArgs.java

@@ -267,4 +267,11 @@ public class CommandLineArgs {
       names = PROPAGATE_DATA_PROVIDER_FAILURES_AS_TEST_FAILURE,
       description = "Should TestNG consider failures in Data Providers  as test failures.")
   public Boolean propagateDataProviderFailureAsTestFailure = false;
+
+  public static final String GENERATE_RESULTS_PER_SUITE = "-generateResultsPerSuite";
+
+  @Parameter(
+      names = GENERATE_RESULTS_PER_SUITE,
+      description = "Should TestNG consider failures in Data Providers  as test failures.")
+  public Boolean generateResultsPerSuite = false;
 }

testng-core/src/main/java/org/testng/TestNG.java

@@ -54,6 +54,7 @@
 import org.testng.reporters.EmailableReporter2;
 import org.testng.reporters.FailedReporter;
 import org.testng.reporters.JUnitReportReporter;
+import org.testng.reporters.PerSuiteXMLReporter;
 import org.testng.reporters.SuiteHTMLReporter;
 import org.testng.reporters.VerboseReporter;
 import org.testng.reporters.XMLReporter;
@@ -811,6 +812,7 @@ public List<ISuiteListener> getSuiteListeners() {
 
   private Boolean m_preserveOrder = XmlSuite.DEFAULT_PRESERVE_ORDER;
   private Boolean m_groupByInstances;
+  private boolean m_generateResultsPerSuite = false;
 
   private IConfiguration m_configuration;
 
@@ -829,6 +831,10 @@ public void setExecutorFactoryClass(String clazzName) {
     this.m_executorFactory = createExecutorFactoryInstanceUsing(clazzName);
   }
 
+  public void setGenerateResultsPerSuite(boolean generateResultsPerSuite) {
+    this.m_generateResultsPerSuite = generateResultsPerSuite;
+  }
+
   private IExecutorFactory createExecutorFactoryInstanceUsing(String clazzName) {
     Class<?> cls = ClassHelper.forName(clazzName);
     Object instance = m_objectFactory.newInstance(cls);
@@ -934,7 +940,11 @@ private void initializeDefaultListeners() {
       addReporter(SuiteHTMLReporter.class);
       addReporter(Main.class);
       addReporter(FailedReporter.class);
-      addReporter(XMLReporter.class);
+      if (m_generateResultsPerSuite) {
+        addReporter(PerSuiteXMLReporter.class);
+      } else {
+        addReporter(XMLReporter.class);
+      }
       if (RuntimeBehavior.useOldTestNGEmailableReporter()) {
         addReporter(EmailableReporter.class);
       } else if (RuntimeBehavior.useEmailableReporter()) {
@@ -1449,6 +1459,9 @@ protected void configure(CommandLineArgs cla) {
     Optional.ofNullable(cla.propagateDataProviderFailureAsTestFailure)
         .ifPresent(value -> propagateDataProviderFailureAsTestFailure());
     setReportAllDataDrivenTestsAsSkipped(cla.includeAllDataDrivenTestsWhenSkipping);
+
+    Optional.ofNullable(cla.generateResultsPerSuite).ifPresent(this::setGenerateResultsPerSuite);
+
     if (cla.verbose != null) {
       setVerbose(cla.verbose);
     }
@@ -1744,6 +1757,14 @@ public void configure(Map cmdLineArgs) {
       result.dependencyInjectorFactoryClass = dependencyInjectorFactoryClass;
     }
 
+    result.ignoreMissedTestNames =
+        Boolean.parseBoolean(
+            cmdLineArgs.getOrDefault(CommandLineArgs.IGNORE_MISSED_TEST_NAMES, false).toString());
+
+    result.generateResultsPerSuite =
+        Boolean.parseBoolean(
+            cmdLineArgs.getOrDefault(CommandLineArgs.GENERATE_RESULTS_PER_SUITE, false).toString());
+
     configure(result);
   }
 

testng-core/src/main/java/org/testng/reporters/AbstractXmlReporter.java

@@ -0,0 +1,284 @@
+package org.testng.reporters;
+
+import java.io.File;
+import java.util.Collection;
+import java.util.Date;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import org.testng.IReporter;
+import org.testng.ISuite;
+import org.testng.ISuiteResult;
+import org.testng.ITestContext;
+import org.testng.ITestNGMethod;
+import org.testng.ITestResult;
+import org.testng.Reporter;
+import org.testng.internal.Utils;
+import org.testng.util.TimeUtils;
+
+public abstract class AbstractXmlReporter implements IReporter, ICustomizeXmlReport {
+
+  private final XMLReporterConfig config = new XMLReporterConfig();
+
+  public String fileName() {
+    return RuntimeBehavior.getDefaultFileNameForXmlReports();
+  }
+
+  @Override
+  public XMLReporterConfig getConfig() {
+    return config;
+  }
+
+  @Override
+  public void addCustomTagsFor(XMLStringBuffer xmlBuffer, ITestResult testResult) {}
+
+  protected final void writeReporterOutput(XMLStringBuffer xmlBuffer) {
+    writeReporterOutput(xmlBuffer, Reporter.getOutput());
+  }
+
+  protected final void writeReporterOutput(XMLStringBuffer xmlBuffer, List<String> output) {
+    // TODO: Cosmin - maybe a <line> element isn't indicated for each line
+    xmlBuffer.push(XMLReporterConfig.TAG_REPORTER_OUTPUT);
+    for (String line : output) {
+      if (line != null) {
+        xmlBuffer.push(XMLReporterConfig.TAG_LINE);
+        xmlBuffer.addCDATA(line);
+        xmlBuffer.pop();
+      }
+    }
+    xmlBuffer.pop();
+  }
+
+  protected final void writeSuite(XMLStringBuffer rootBuffer, ISuite suite) {
+    switch (config.getFileFragmentationLevel()) {
+      case XMLReporterConfig.FF_LEVEL_NONE:
+        writeSuiteToBuffer(rootBuffer, suite);
+        break;
+      case XMLReporterConfig.FF_LEVEL_SUITE:
+      case XMLReporterConfig.FF_LEVEL_SUITE_RESULT:
+        File suiteFile = referenceSuite(rootBuffer, suite);
+        writeSuiteToFile(suiteFile, suite);
+        break;
+      default:
+        throw new AssertionError("Unexpected value: " + config.getFileFragmentationLevel());
+    }
+  }
+
+  private void writeSuiteToFile(File suiteFile, ISuite suite) {
+    XMLStringBuffer xmlBuffer = new XMLStringBuffer();
+    writeSuiteToBuffer(xmlBuffer, suite);
+    File parentDir = suiteFile.getParentFile();
+    suiteFile.getParentFile().mkdirs();
+    if (parentDir.exists() || suiteFile.getParentFile().exists()) {
+      Utils.writeUtf8File(parentDir.getAbsolutePath(), fileName(), xmlBuffer.toXML());
+    }
+  }
+
+  private File referenceSuite(XMLStringBuffer xmlBuffer, ISuite suite) {
+    String relativePath = suite.getName() + File.separatorChar + fileName();
+    File suiteFile = new File(config.getOutputDirectory(), relativePath);
+    Properties attrs = new Properties();
+    attrs.setProperty(XMLReporterConfig.ATTR_URL, relativePath);
+    xmlBuffer.addEmptyElement(XMLReporterConfig.TAG_SUITE, attrs);
+    return suiteFile;
+  }
+
+  private void writeSuiteToBuffer(XMLStringBuffer xmlBuffer, ISuite suite) {
+    xmlBuffer.push(XMLReporterConfig.TAG_SUITE, getSuiteAttributes(suite));
+    writeSuiteGroups(xmlBuffer, suite);
+
+    Map<String, ISuiteResult> results = suite.getResults();
+    XMLSuiteResultWriter suiteResultWriter = new XMLSuiteResultWriter(config, this);
+    for (Map.Entry<String, ISuiteResult> result : results.entrySet()) {
+      suiteResultWriter.writeSuiteResult(xmlBuffer, result.getValue());
+    }
+
+    xmlBuffer.pop();
+  }
+
+  private Set<ITestNGMethod> getUniqueMethodSet(Collection<ITestNGMethod> methods) {
+    return new LinkedHashSet<>(methods);
+  }
+
+  private void writeSuiteGroups(XMLStringBuffer xmlBuffer, ISuite suite) {
+    xmlBuffer.push(XMLReporterConfig.TAG_GROUPS);
+    Map<String, Collection<ITestNGMethod>> methodsByGroups = suite.getMethodsByGroups();
+    for (Map.Entry<String, Collection<ITestNGMethod>> entry : methodsByGroups.entrySet()) {
+      Properties groupAttrs = new Properties();
+      groupAttrs.setProperty(XMLReporterConfig.ATTR_NAME, entry.getKey());
+      xmlBuffer.push(XMLReporterConfig.TAG_GROUP, groupAttrs);
+      Set<ITestNGMethod> groupMethods = getUniqueMethodSet(entry.getValue());
+      for (ITestNGMethod groupMethod : groupMethods) {
+        Properties methodAttrs = new Properties();
+        methodAttrs.setProperty(XMLReporterConfig.ATTR_NAME, groupMethod.getMethodName());
+        methodAttrs.setProperty(XMLReporterConfig.ATTR_METHOD_SIG, groupMethod.toString());
+        methodAttrs.setProperty(XMLReporterConfig.ATTR_CLASS, groupMethod.getRealClass().getName());
+        xmlBuffer.addEmptyElement(XMLReporterConfig.TAG_METHOD, methodAttrs);
+      }
+      xmlBuffer.pop();
+    }
+    xmlBuffer.pop();
+  }
+
+  private Properties getSuiteAttributes(ISuite suite) {
+    Properties props = new Properties();
+    props.setProperty(XMLReporterConfig.ATTR_NAME, suite.getName());
+
+    // Calculate the duration
+    Map<String, ISuiteResult> results = suite.getResults();
+    Date minStartDate = new Date();
+    Date maxEndDate = null;
+    // TODO: We could probably optimize this in order not to traverse this twice
+    for (Map.Entry<String, ISuiteResult> result : results.entrySet()) {
+      ITestContext testContext = result.getValue().getTestContext();
+      Date startDate = testContext.getStartDate();
+      Date endDate = testContext.getEndDate();
+      if (minStartDate.after(startDate)) {
+        minStartDate = startDate;
+      }
+      if (maxEndDate == null || maxEndDate.before(endDate)) {
+        maxEndDate = endDate != null ? endDate : startDate;
+      }
+    }
+    // The suite could be completely empty
+    if (maxEndDate == null) {
+      maxEndDate = minStartDate;
+    }
+    setDurationAttributes(config, props, minStartDate, maxEndDate);
+    return props;
+  }
+
+  protected static void setDurationAttributes(
+      XMLReporterConfig config, Properties attributes, Date minStartDate, Date maxEndDate) {
+
+    String startTime =
+        TimeUtils.formatTimeInLocalOrSpecifiedTimeZone(
+            minStartDate.getTime(), config.getTimestampFormat());
+    String endTime =
+        TimeUtils.formatTimeInLocalOrSpecifiedTimeZone(
+            maxEndDate.getTime(), config.getTimestampFormat());
+    long duration = maxEndDate.getTime() - minStartDate.getTime();
+
+    attributes.setProperty(XMLReporterConfig.ATTR_STARTED_AT, startTime);
+    attributes.setProperty(XMLReporterConfig.ATTR_FINISHED_AT, endTime);
+    attributes.setProperty(XMLReporterConfig.ATTR_DURATION_MS, Long.toString(duration));
+  }
+
+  protected final Properties writeSummaryCount(Count count, XMLStringBuffer rootBuffer) {
+    Properties p = new Properties();
+    p.put("passed", count.passed);
+    p.put("failed", count.failed);
+    p.put("skipped", count.skipped);
+    if (count.retried > 0) {
+      p.put("retried", count.retried);
+    }
+    p.put("ignored", count.ignored);
+    p.put("total", count.total());
+    return p;
+  }
+
+  protected final Count computeCountForSuite(ISuite s) {
+    int passed = 0;
+    int failed = 0;
+    int skipped = 0;
+    int retried = 0;
+    int ignored = 0;
+    for (ISuiteResult sr : s.getResults().values()) {
+      ITestContext testContext = sr.getTestContext();
+      passed += testContext.getPassedTests().size();
+      failed += testContext.getFailedTests().size();
+      int retriedPerTest = 0;
+      int skippedPerTest = 0;
+      for (ITestResult result : testContext.getSkippedTests().getAllResults()) {
+        if (result.wasRetried()) {
+          retriedPerTest++;
+        } else {
+          skippedPerTest++;
+        }
+      }
+      skipped += skippedPerTest;
+      retried += retriedPerTest;
+      ignored += testContext.getExcludedMethods().stream().filter(ITestNGMethod::isTest).count();
+    }
+    return Count.Builder.builder()
+        .withPassed(passed)
+        .withFailed(failed)
+        .withSkipped(skipped)
+        .withRetried(retried)
+        .withIgnored(ignored)
+        .build();
+  }
+
+  public static class Count {
+    private int passed;
+    private int failed;
+    private int skipped;
+    private int retried;
+    private int ignored;
+
+    public int total() {
+      return passed + failed + skipped + retried + ignored;
+    }
+
+    public void add(Count count) {
+      this.passed += count.passed;
+      this.failed += count.failed;
+      this.skipped += count.skipped;
+      this.retried += count.retried;
+      this.ignored += count.retried;
+    }
+
+    private Count(Builder builder) {
+      passed = builder.passed;
+      failed = builder.failed;
+      skipped = builder.skipped;
+      retried = builder.retried;
+      ignored = builder.ignored;
+    }
+
+    public static final class Builder {
+      private int passed;
+      private int failed;
+      private int skipped;
+      private int retried;
+      private int ignored;
+
+      private Builder() {}
+
+      public static Builder builder() {
+        return new Builder();
+      }
+
+      public Builder withPassed(int val) {
+        passed = val;
+        return this;
+      }
+
+      public Builder withFailed(int val) {
+        failed = val;
+        return this;
+      }
+
+      public Builder withSkipped(int val) {
+        skipped = val;
+        return this;
+      }
+
+      public Builder withRetried(int val) {
+        retried = val;
+        return this;
+      }
+
+      public Builder withIgnored(int val) {
+        ignored = val;
+        return this;
+      }
+
+      public Count build() {
+        return new Count(this);
+      }
+    }
+  }
+}