Extract xUnit test results parser

Summary:
Many test frameworks can format their output in xUnit-like format.

Test Plan: Tested the Nose engine with a Nose one, and the pytest with a demo project.

Reviewers: epriestley

Reviewed By: epriestley

CC: Korvin, aran

Differential Revision: https://secure.phabricator.com/D7011

Conflicts:
	src/__phutil_library_map__.php

src/__phutil_library_map__.php

@@ -163,6 +163,7 @@ phutil_register_library_map(array(
     'ArcanistXHPASTLintTestSwitchHook' => 'lint/linter/__tests__/ArcanistXHPASTLintTestSwitchHook.php',
     'ArcanistXHPASTLinter' => 'lint/linter/ArcanistXHPASTLinter.php',
     'ArcanistXHPASTLinterTestCase' => 'lint/linter/__tests__/ArcanistXHPASTLinterTestCase.php',
+    'ArcanistXUnitTestResultParser' => 'unit/engine/ArcanistXUnitTestResultParser.php',
     'CSharpToolsTestEngine' => 'unit/engine/CSharpToolsTestEngine.php',
     'ComprehensiveLintEngine' => 'lint/engine/ComprehensiveLintEngine.php',
     'ExampleLintEngine' => 'lint/engine/ExampleLintEngine.php',
@@ -178,6 +179,7 @@ phutil_register_library_map(array(
     'PytestTestEngine' => 'unit/engine/PytestTestEngine.php',
     'UnitTestableArcanistLintEngine' => 'lint/engine/UnitTestableArcanistLintEngine.php',
     'XUnitTestEngine' => 'unit/engine/XUnitTestEngine.php',
+    'XUnitTestResultParserTestCase' => 'unit/engine/__tests__/XUnitTestResultParserTestCase.php',
   ),
   'function' =>
   array(
@@ -318,5 +320,6 @@ phutil_register_library_map(array(
     'PytestTestEngine' => 'ArcanistBaseUnitTestEngine',
     'UnitTestableArcanistLintEngine' => 'ArcanistLintEngine',
     'XUnitTestEngine' => 'ArcanistBaseUnitTestEngine',
+    'XUnitTestResultParserTestCase' => 'ArcanistTestCase',
   ),
 ));

src/unit/engine/ArcanistXUnitTestResultParser.php

@@ -0,0 +1,97 @@
+<?php
+
+/**
+ * Parser for JUnit, NUnit, etc results format - https://gist.github.com/959290
+ *
+ * @group unitrun
+ */
+final class ArcanistXUnitTestResultParser {
+
+  /**
+   * Parse test results from provided input and return an array
+   * of ArcanistUnitTestResult
+   *
+   * @param string $path Path to test (Ignored)
+   * @param string $test_results String containing test results
+   *
+   * @return array ArcanistUnitTestResult
+   */
+  public function parseTestResults($test_results) {
+    if (!strlen($test_results)) {
+      throw new Exception(
+        'test_results argument to parseTestResults must not be empty');
+    }
+
+    // xunit xsd: https://gist.github.com/959290
+    $xunit_dom = new DOMDocument();
+    $load_success = @$xunit_dom->loadXML($test_results);
+
+    if (!$load_success) {
+      $input_start = phutil_utf8_shorten($test_results, 150);
+      throw new Exception(
+        "Failed to load XUnit report; Input starts with:\n\n {$input_start}");
+    }
+
+    $results = array();
+    $testcases = $xunit_dom->getElementsByTagName("testcase");
+    foreach ($testcases as $testcase) {
+      $classname = $testcase->getAttribute("classname");
+      $name = $testcase->getAttribute("name");
+      $time = $testcase->getAttribute("time");
+
+      $status = ArcanistUnitTestResult::RESULT_PASS;
+      $user_data = "";
+
+      // A skipped test is a test which was ignored using framework
+      // mechanizms (e.g. @skip decorator)
+      $skipped = $testcase->getElementsByTagName("skipped");
+      if ($skipped->length > 0) {
+        $status = ArcanistUnitTestResult::RESULT_SKIP;
+        $messages = array();
+        for ($ii = 0; $ii < $skipped->length; $ii++) {
+          $messages[] = trim($skipped->item($ii)->nodeValue, " \n");
+        }
+
+        $user_data .= implode("\n", $messages);
+      }
+
+      // Failure is a test which the code has explicitly failed by using
+      // the mechanizms for that purpose. e.g., via an assertEquals
+      $failures = $testcase->getElementsByTagName("failure");
+      if ($failures->length > 0) {
+        $status = ArcanistUnitTestResult::RESULT_FAIL;
+        $messages = array();
+        for ($ii = 0; $ii < $failures->length; $ii++) {
+          $messages[] = trim($failures->item($ii)->nodeValue, " \n");
+        }
+
+        $user_data .= implode("\n", $messages)."\n";
+      }
+
+      // An errored test is one that had an unanticipated problem. e.g., an
+      // unchecked throwable, or a problem with an implementation of the
+      // test.
+      $errors = $testcase->getElementsByTagName("error");
+      if ($errors->length > 0) {
+        $status = ArcanistUnitTestResult::RESULT_BROKEN;
+        $messages = array();
+        for ($ii = 0; $ii < $errors->length; $ii++) {
+          $messages[] = trim($errors->item($ii)->nodeValue, " \n");
+        }
+
+        $user_data .= implode("\n", $messages)."\n";
+      }
+
+      $result = new ArcanistUnitTestResult();
+      $result->setName($classname.".".$name);
+      $result->setResult($status);
+      $result->setDuration($time);
+      $result->setUserData($user_data);
+
+      $results[] = $result;
+    }
+
+    return $results;
+  }
+
+}

src/unit/engine/NoseTestEngine.php

@@ -77,6 +77,7 @@ final class NoseTestEngine extends ArcanistBaseUnitTestEngine {
       $xunit_tmp = $tmpfiles[$test_path]['xunit'];
       $cover_tmp = $tmpfiles[$test_path]['cover'];
 
+      $this->parser = new ArcanistXUnitTestResultParser();
       $results[] = $this->parseTestResults($source_path,
                                            $xunit_tmp,
                                            $cover_tmp);
@@ -99,74 +100,15 @@ final class NoseTestEngine extends ArcanistBaseUnitTestEngine {
   }
 
   public function parseTestResults($source_path, $xunit_tmp, $cover_tmp) {
-    // xunit xsd: https://gist.github.com/959290
+    $results = $this->parser->parseTestResults(
-    $xunit_dom = new DOMDocument();
+      Filesystem::readFile($xunit_tmp));
-    $xunit_dom->loadXML(Filesystem::readFile($xunit_tmp));
 
     // coverage is for all testcases in the executed $path
-    $coverage = array();
     if ($this->getEnableCoverage() !== false) {
       $coverage = $this->readCoverage($cover_tmp, $source_path);
-    }
+      foreach ($results as $result) {
-
+        $result->setCoverage($coverage);
-    $results = array();
-    $testcases = $xunit_dom->getElementsByTagName("testcase");
-    foreach ($testcases as $testcase) {
-      $classname = $testcase->getAttribute("classname");
-      $name = $testcase->getAttribute("name");
-      $time = $testcase->getAttribute("time");
-
-      $status = ArcanistUnitTestResult::RESULT_PASS;
-      $user_data = "";
-
-      // A skipped test is a test which was ignored using framework
-      // mechanizms (e.g. @skip decorator)
-      $skipped = $testcase->getElementsByTagName("skipped");
-      if ($skipped->length > 0) {
-        $status = ArcanistUnitTestResult::RESULT_SKIP;
-        $messages = array();
-        for ($ii = 0; $ii < $skipped->length; $ii++) {
-          $messages[] = trim($skipped->item($ii)->nodeValue, " \n");
-        }
-
-        $user_data .= implode("\n", $messages);
       }
-
-      // Failure is a test which the code has explicitly failed by using
-      // the mechanizms for that purpose. e.g., via an assertEquals
-      $failures = $testcase->getElementsByTagName("failure");
-      if ($failures->length > 0) {
-        $status = ArcanistUnitTestResult::RESULT_FAIL;
-        $messages = array();
-        for ($ii = 0; $ii < $failures->length; $ii++) {
-          $messages[] = trim($failures->item($ii)->nodeValue, " \n");
-        }
-
-        $user_data .= implode("\n", $messages)."\n";
-      }
-
-      // An errored test is one that had an unanticipated problem. e.g., an
-      // unchecked throwable, or a problem with an implementation of the
-      // test.
-      $errors = $testcase->getElementsByTagName("error");
-      if ($errors->length > 0) {
-        $status = ArcanistUnitTestResult::RESULT_BROKEN;
-        $messages = array();
-        for ($ii = 0; $ii < $errors->length; $ii++) {
-          $messages[] = trim($errors->item($ii)->nodeValue, " \n");
-        }
-
-        $user_data .= implode("\n", $messages)."\n";
-      }
-
-      $result = new ArcanistUnitTestResult();
-      $result->setName($classname.".".$name);
-      $result->setResult($status);
-      $result->setDuration($time);
-      $result->setCoverage($coverage);
-      $result->setUserData($user_data);
-
-      $results[] = $result;
     }
 
     return $results;

src/unit/engine/PytestTestEngine.php

@@ -12,78 +12,12 @@ final class PytestTestEngine extends ArcanistBaseUnitTestEngine {
 
     $cmd_line = csprintf('py.test --junitxml=%s',
                          $junit_tmp);
-
     $future = new ExecFuture('%C', $cmd_line);
     list($stdout, $stderr) = $future->resolvex();
 
-    return $this->parseTestResults($junit_tmp);
+    $parser = new ArcanistXUnitTestResultParser();
-  }
 
-  public function parseTestResults($junit_tmp) {
+    return $parser->parseTestResults(Filesystem::readFile($junit_tmp));
-    // xunit xsd: https://gist.github.com/959290
-    $xunit_dom = new DOMDocument();
-    $xunit_dom->loadXML(Filesystem::readFile($junit_tmp));
-
-    $results = array();
-    $testcases = $xunit_dom->getElementsByTagName("testcase");
-    foreach ($testcases as $testcase) {
-      $classname = $testcase->getAttribute("classname");
-      $name = $testcase->getAttribute("name");
-      $time = $testcase->getAttribute("time");
-
-      $status = ArcanistUnitTestResult::RESULT_PASS;
-      $user_data = "";
-
-      // A skipped test is a test which was ignored using framework
-      // mechanizms (e.g. @skip decorator)
-      $skipped = $testcase->getElementsByTagName("skipped");
-      if ($skipped->length > 0) {
-        $status = ArcanistUnitTestResult::RESULT_SKIP;
-        $messages = array();
-        for ($ii = 0; $ii < $skipped->length; $ii++) {
-          $messages[] = trim($skipped->item($ii)->nodeValue, " \n");
-        }
-
-        $user_data .= implode("\n", $messages);
-      }
-
-      // Failure is a test which the code has explicitly failed by using
-      // the mechanizms for that purpose. e.g., via an assertEquals
-      $failures = $testcase->getElementsByTagName("failure");
-      if ($failures->length > 0) {
-        $status = ArcanistUnitTestResult::RESULT_FAIL;
-        $messages = array();
-        for ($ii = 0; $ii < $failures->length; $ii++) {
-          $messages[] = trim($failures->item($ii)->nodeValue, " \n");
-        }
-
-        $user_data .= implode("\n", $messages)."\n";
-      }
-
-      // An errored test is one that had an unanticipated problem. e.g., an
-      // unchecked throwable, or a problem with an implementation of the
-      // test.
-      $errors = $testcase->getElementsByTagName("error");
-      if ($errors->length > 0) {
-        $status = ArcanistUnitTestResult::RESULT_BROKEN;
-        $messages = array();
-        for ($ii = 0; $ii < $errors->length; $ii++) {
-          $messages[] = trim($errors->item($ii)->nodeValue, " \n");
-        }
-
-        $user_data .= implode("\n", $messages)."\n";
-      }
-
-      $result = new ArcanistUnitTestResult();
-      $result->setName($classname.".".$name);
-      $result->setResult($status);
-      $result->setDuration($time);
-      $result->setUserData($user_data);
-
-      $results[] = $result;
-    }
-
-    return $results;
   }
 
 }

src/unit/engine/__tests__/XUnitTestResultParserTestCase.php

@@ -0,0 +1,55 @@
+<?php
+
+/**
+ * Test for @{class:ArcanistXUnitTestResultParser}.
+ *
+ * (putting tests in your tests so you can test
+ * while you test)
+ *
+ * @group testcase
+ */
+final class XUnitTestResultParserTestCase extends ArcanistTestCase {
+
+  public function testAcceptsNoTestsInput() {
+    $stubbed_results = Filesystem::readFile(
+      dirname(__FILE__).'/testresults/xunit.no-tests');
+    $parsed_results = id(new ArcanistXUnitTestResultParser())
+      ->parseTestResults($stubbed_results);
+
+    $this->assertEqual(0, count($parsed_results));
+  }
+
+  public function testAcceptsSimpleInput() {
+    $stubbed_results = Filesystem::readFile(
+      dirname(__FILE__).'/testresults/xunit.simple');
+    $parsed_results = id(new ArcanistXUnitTestResultParser())
+      ->parseTestResults($stubbed_results);
+
+    $this->assertEqual(3, count($parsed_results));
+  }
+
+  public function testEmptyInputFailure() {
+    try {
+      $parsed_results = id(new ArcanistXUnitTestResultParser())
+        ->parseTestResults('');
+
+      $this->failTest('Should throw on empty input');
+    } catch (Exception $e) {
+      // OK
+    }
+  }
+
+  public function testInvalidXmlInputFailure() {
+    $stubbed_results = Filesystem::readFile(
+      dirname(__FILE__).'/testresults/xunit.invallid-xml');
+    try {
+      $parsed_results = id(new ArcanistXUnitTestResultParser())
+        ->parseTestResults($stubbed_results);
+
+      $this->failTest('Should throw on non-xml input');
+    } catch (Exception $e) {
+      // OK
+    }
+  }
+
+}

src/unit/engine/__tests__/testresults/xunit.invallid-xml

@@ -0,0 +1 @@
+something else

src/unit/engine/__tests__/testresults/xunit.no-tests

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="utf-8"?><testsuite name="" errors="0" failures="0" skips="0" tests="0" time="0.001" ></testsuite>

src/unit/engine/__tests__/testresults/xunit.simple

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?>
+<testsuite name="" errors="0" failures="1" skips="0" tests="3" time="0.019">
+  <testcase classname="forpytest.tests" name="test_answer" time="0.0110499858856">
+    <failure message="test failure">def test_answer():
+&gt;       assert func(3) == 5
+E       assert 4 == 5
+E        +  where 4 = func(3)
+
+tests.py:5: AssertionError</failure>
+  </testcase>
+  <testcase classname="forpytest.tests" name="test_something" time="6.29425048828e-05"/>
+  <testcase classname="forpytest.tests" name="test_nothing" time="5.91278076172e-05"/>
+</testsuite>