Added base test result parser class. Implemented Go test result parser based on Go's built in test utility output.

Test Plan: Ran PhpunitTestEngine unit test and used both test result parsers to generate test results.

Reviewers: epriestley

Reviewed By: epriestley

CC: aran, Korvin, aurelijus

Differential Revision: https://secure.phabricator.com/D4676

src/__phutil_library_map__.php

@@ -18,6 +18,7 @@ phutil_register_library_map(array(
     'ArcanistArcanistLinterTestCase' => 'lint/linter/__tests__/ArcanistArcanistLinterTestCase.php',
     'ArcanistBaseCommitParser' => 'parser/ArcanistBaseCommitParser.php',
     'ArcanistBaseCommitParserTestCase' => 'parser/__tests__/ArcanistBaseCommitParserTestCase.php',
+    'ArcanistBaseTestResultParser' => 'unit/engine/ArcanistBaseTestResultParser.php',
     'ArcanistBaseUnitTestEngine' => 'unit/engine/ArcanistBaseUnitTestEngine.php',
     'ArcanistBaseWorkflow' => 'workflow/ArcanistBaseWorkflow.php',
     'ArcanistBranchWorkflow' => 'workflow/ArcanistBranchWorkflow.php',
@@ -143,6 +144,8 @@ phutil_register_library_map(array(
     'ArcanistXHPASTLinterTestCase' => 'lint/linter/__tests__/ArcanistXHPASTLinterTestCase.php',
     'ComprehensiveLintEngine' => 'lint/engine/ComprehensiveLintEngine.php',
     'ExampleLintEngine' => 'lint/engine/ExampleLintEngine.php',
+    'GoTestResultParser' => 'unit/engine/GoTestResultParser.php',
+    'GoTestResultParserTestCase' => 'unit/engine/__tests__/GoTestResultParserTestCase.php',
     'NoseTestEngine' => 'unit/engine/NoseTestEngine.php',
     'PHPUnitTestEngineTestCase' => 'unit/engine/__tests__/PHPUnitTestEngineTestCase.php',
     'PhpunitResultParser' => 'unit/engine/PhpunitResultParser.php',
@@ -259,8 +262,11 @@ phutil_register_library_map(array(
     'ArcanistXHPASTLinterTestCase' => 'ArcanistArcanistLinterTestCase',
     'ComprehensiveLintEngine' => 'ArcanistLintEngine',
     'ExampleLintEngine' => 'ArcanistLintEngine',
+    'GoTestResultParser' => 'ArcanistBaseTestResultParser',
+    'GoTestResultParserTestCase' => 'ArcanistTestCase',
     'NoseTestEngine' => 'ArcanistBaseUnitTestEngine',
     'PHPUnitTestEngineTestCase' => 'ArcanistTestCase',
+    'PhpunitResultParser' => 'ArcanistBaseTestResultParser',
     'PhpunitTestEngine' => 'ArcanistBaseUnitTestEngine',
     'PhutilLintEngine' => 'ArcanistLintEngine',
     'PhutilUnitTestEngine' => 'ArcanistBaseUnitTestEngine',

src/unit/engine/ArcanistBaseTestResultParser.php

@@ -0,0 +1,46 @@
+<?php
+
+/**
+ * Abstract Base class for test result parsers
+ */
+abstract class ArcanistBaseTestResultParser {
+
+  protected $enableCoverage;
+  protected $projectRoot;
+  protected $coverateFile;
+
+  public function setEnableCoverage($enable_coverage) {
+    $this->enableCoverage = $enable_coverage;
+
+    return $this;
+  }
+
+  public function setProjectRoot($project_root) {
+    $this->projectRoot = $project_root;
+
+    return $this;
+  }
+
+  public function setCoverageFile($coverage_file) {
+    $this->coverageFile = $coverage_file;
+
+    return $this;
+  }
+
+  public function setAffectedTests($affected_tests) {
+    $this->affectedTests = $affected_tests;
+
+    return $this;
+  }
+
+  /**
+   * Parse test results from provided input and return an array
+   * of ArcanistUnitTestResult
+   *
+   * @param string $path Path to test
+   * @param string $test_results String containing test results
+   *
+   * @return array ArcanistUnitTestResult
+   */
+  abstract public function parseTestResults($path, $test_results);
+}

src/unit/engine/GoTestResultParser.php

@@ -0,0 +1,140 @@
+<?php
+
+/**
+ * Go Test Result Parsing utility
+ *
+ * Intended to enable custom unit engines derived
+ * from Go's built-in test utility to reuse
+ * common business logic related to parsing
+ * Go test results.
+ *
+ * (To generate test output, run something like:
+ * `go test -v`)
+ */
+final class GoTestResultParser extends ArcanistBaseTestResultParser {
+
+  /**
+   * Parse test results from Go test report
+   * (e.g. `go test -v`)
+   *
+   * @param string $path Path to test
+   * @param string $test_results String containing Go test output
+   *
+   * @return array
+   */
+  public function parseTestResults($path, $test_results) {
+
+    $test_results = explode("\n", $test_results);
+
+    $results = array();
+    // We'll get our full test case name at the end and add it back in
+    $test_case_name = "";
+
+    // Temp store for test case results (in case we run multiple test cases)
+    $test_case_results = array();
+    foreach ($test_results as $i => $line) {
+
+      if (strncmp($line, "--- PASS", 8) === 0) {
+        // We have a passing test
+        $meta = array();
+        preg_match(
+          '/^--- PASS: (?P<test_name>.+) \((?P<time>.+) seconds\).*/',
+          $line,
+          $meta);
+
+        $result = new ArcanistUnitTestResult();
+        // For now set name without test case, we'll add it later
+        $result->setName($meta['test_name']);
+        $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
+        $result->setDuration($meta['time']);
+
+        $test_case_results[] = $result;
+
+        continue;
+      }
+
+      if (strncmp($line, "--- FAIL", 8) === 0) {
+        // We have a failing test
+        $reason = trim($test_results[$i + 1]);
+        $meta = array();
+        preg_match(
+          '/^--- FAIL: (?P<test_name>.+) \((?P<time>.+) seconds\).*/',
+          $line,
+          $meta);
+
+        $result = new ArcanistUnitTestResult();
+        $result->setName($meta['test_name']);
+        $result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
+        $result->setDuration($meta['time']);
+        $result->setUserData($reason."\n");
+
+        $test_case_results[] = $result;
+
+        continue;
+      }
+
+      if (strncmp($line, "ok", 2) === 0) {
+        $meta = array();
+        preg_match(
+          '/^ok[\s\t]+(?P<test_name>\w.*)[\s\t]+(?P<time>.*)s.*/',
+          $line,
+          $meta);
+
+        $test_case_name = str_replace("/", "::", $meta['test_name']);
+
+        // Our test case passed
+        // check to make sure we were in verbose (-v) mode
+        if (empty($test_case_results)) {
+          // We weren't in verbose mode
+          // create one successful result for the whole test case
+          $test_name = "Go::TestCase::".$test_case_name;
+
+          $result = new ArcanistUnitTestResult();
+          $result->setName($test_name);
+          $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
+          $result->setDuration($meta['time']);
+
+          $results[] = $result;
+        } else {
+          $test_case_results = $this->fixNames(
+            $test_case_results,
+            $test_case_name);
+          $results = array_merge($results, $test_case_results);
+          $test_case_results = array();
+        }
+
+        continue;
+      }
+
+      if (strncmp($line, "FAIL\t", 5) === 0) {
+        $meta = array();
+        preg_match(
+          '/^FAIL[\s\t]+(?P<test_name>\w.*)[\s\t]+.*/',
+          $line,
+          $meta);
+
+        $test_case_name = str_replace("/", "::", $meta['test_name']);
+
+        $test_case_results = $this->fixNames(
+          $test_case_results,
+          $test_case_name);
+        $results = array_merge($results, $test_case_results);
+        $test_case_results = array();
+
+        continue;
+      }
+    }
+
+    return $results;
+  }
+
+  private function fixNames($test_case_results, $test_case_name) {
+
+    foreach ($test_case_results as &$result) {
+      $test_name = $result->getName();
+      $result->setName("Go::Test::".$test_case_name."::".$test_name);
+    }
+
+    return $test_case_results;
+  }
+}

src/unit/engine/PhpunitResultParser.php

@@ -11,105 +11,79 @@
  * engine, see PhpunitTestEngine.
  *
  */
-final class PhpunitResultParser {
-
-  private $enableCoverage;
-  private $projectRoot;
-
-  public function setEnableCoverage($enable_coverage) {
-    $this->enableCoverage = $enable_coverage;
-
-    return $this;
-  }
-
-  public function setProjectRoot($project_root) {
-    $this->projectRoot = $project_root;
-
-    return $this;
-  }
+final class PhpunitResultParser extends ArcanistBaseTestResultParser {
 
   /**
    * Parse test results from phpunit json report
    *
    * @param string $path Path to test
-   * @param string $json_path Path to phpunit json report
-   * @param string $clover_tmp Path to phpunit clover report
-   * @param array $affected_tests Array of the tests affected by this run
-   * @param bool $enable_coverage Option to enable coverage in results
+   * @param string $test_results String containing phpunit json report
    *
    * @return array
    */
-  public function parseTestResults(
-    $path,
-    $json_tmp,
-    $clover_tmp,
-    $affected_tests) {
+  public function parseTestResults($path, $test_results) {
 
-      $test_results = Filesystem::readFile($json_tmp);
+    $report = $this->getJsonReport($test_results);
 
-      $report = $this->getJsonReport($json_tmp);
+    // coverage is for all testcases in the executed $path
+    $coverage = array();
+    if ($this->enableCoverage !== false) {
+      $coverage = $this->readCoverage();
+    }
 
-      // coverage is for all testcases in the executed $path
-      $coverage = array();
-      if ($this->enableCoverage !== false) {
-        $coverage = $this->readCoverage($clover_tmp, $affected_tests);
+    $results = array();
+    foreach ($report as $event) {
+      if ('test' != $event->event) {
+        continue;
       }
 
-      $results = array();
-      foreach ($report as $event) {
-        if ('test' != $event->event) {
-          continue;
+      $status = ArcanistUnitTestResult::RESULT_PASS;
+      $user_data = '';
+
+      if ('fail' == $event->status) {
+        $status = ArcanistUnitTestResult::RESULT_FAIL;
+        $user_data  .= $event->message . "\n";
+        foreach ($event->trace as $trace) {
+          $user_data .= sprintf("\n%s:%s", $trace->file, $trace->line);
         }
-
-        $status = ArcanistUnitTestResult::RESULT_PASS;
-        $user_data = '';
-
-        if ('fail' == $event->status) {
-          $status = ArcanistUnitTestResult::RESULT_FAIL;
-          $user_data  .= $event->message . "\n";
+      } else if ('error' == $event->status) {
+        if (strpos($event->message, 'Skipped Test') !== false) {
+          $status = ArcanistUnitTestResult::RESULT_SKIP;
+          $user_data .= $event->message;
+        } else if (strpos($event->message, 'Incomplete Test') !== false) {
+          $status = ArcanistUnitTestResult::RESULT_SKIP;
+          $user_data .= $event->message;
+        } else {
+          $status = ArcanistUnitTestResult::RESULT_BROKEN;
+          $user_data  .= $event->message;
           foreach ($event->trace as $trace) {
             $user_data .= sprintf("\n%s:%s", $trace->file, $trace->line);
           }
-        } else if ('error' == $event->status) {
-          if (strpos($event->message, 'Skipped Test') !== false) {
-            $status = ArcanistUnitTestResult::RESULT_SKIP;
-            $user_data .= $event->message;
-          } else if (strpos($event->message, 'Incomplete Test') !== false) {
-            $status = ArcanistUnitTestResult::RESULT_SKIP;
-            $user_data .= $event->message;
-          } else {
-            $status = ArcanistUnitTestResult::RESULT_BROKEN;
-            $user_data  .= $event->message;
-            foreach ($event->trace as $trace) {
-              $user_data .= sprintf("\n%s:%s", $trace->file, $trace->line);
-            }
-          }
         }
-
-        $name = preg_replace('/ \(.*\)/', '', $event->test);
-
-        $result = new ArcanistUnitTestResult();
-        $result->setName($name);
-        $result->setResult($status);
-        $result->setDuration($event->time);
-        $result->setCoverage($coverage);
-        $result->setUserData($user_data);
-
-        $results[] = $result;
       }
 
-      return $results;
+      $name = preg_replace('/ \(.*\)/', '', $event->test);
+
+      $result = new ArcanistUnitTestResult();
+      $result->setName($name);
+      $result->setResult($status);
+      $result->setDuration($event->time);
+      $result->setCoverage($coverage);
+      $result->setUserData($user_data);
+
+      $results[] = $result;
     }
 
+    return $results;
+  }
+
   /**
    * Read the coverage from phpunit generated clover report
    *
-   * @param string $path Path to report
-   *
    * @return array
    */
-  private function readCoverage($path, $affected_tests) {
-    $test_results = Filesystem::readFile($path);
+  private function readCoverage() {
+    $test_results = Filesystem::readFile($this->coverageFile);
     if (empty($test_results)) {
       throw new Exception('Clover coverage XML report file is empty, '
         . 'it probably means that phpunit failed to run tests. '
@@ -127,10 +101,10 @@ final class PhpunitResultParser {
 
     foreach ($files as $file) {
       $class_path = $file->getAttribute('name');
-      if (empty($affected_tests[$class_path])) {
+      if (empty($this->affectedTests[$class_path])) {
         continue;
       }
-      $test_path = $affected_tests[$file->getAttribute('name')];
+      $test_path = $this->affectedTests[$file->getAttribute('name')];
       // get total line count in file
       $line_count = count(file($class_path));
 
@@ -172,12 +146,11 @@ final class PhpunitResultParser {
    * We need this non-sense to make json generated by phpunit
    * valid.
    *
-   * @param string $json_tmp Path to JSON report
+   * @param string $json String containing JSON report
    *
    * @return array JSON decoded array
    */
-  private function getJsonReport($json_tmp) {
-    $json = Filesystem::readFile($json_tmp);
+  private function getJsonReport($json) {
 
     if (empty($json)) {
       throw new Exception('JSON report file is empty, '

src/unit/engine/PhpunitTestEngine.php

@@ -100,10 +100,13 @@ final class PhpunitTestEngine extends ArcanistBaseUnitTestEngine {
    * @return array
    */
   private function parseTestResults($path, $json_tmp, $clover_tmp) {
+    $test_results = Filesystem::readFile($json_tmp);
     return id(new PhpunitResultParser())
       ->setEnableCoverage($this->getEnableCoverage())
       ->setProjectRoot($this->projectRoot)
-      ->parseTestResults($path, $json_tmp, $clover_tmp, $this->affectedTests);
+      ->setCoverageFile($clover_tmp)
+      ->setAffectedTests($this->affectedTests)
+      ->parseTestResults($path, $test_results);
   }
 
 

src/unit/engine/__tests__/GoTestResultParserTestCase.php

@@ -0,0 +1,110 @@
+<?php
+
+/**
+ * Test for @{class:GoTestResultParser}.
+ *
+ * (putting tests in your tests so you can test
+ * while you test)
+ *
+ * @group testcase
+ */
+final class GoTestResultParserTestCase extends ArcanistTestCase {
+
+  public function testSingleTestCaseSuccessful() {
+    $stubbed_results = Filesystem::readFile(
+      dirname(__FILE__).'/testresults/go.single-test-case-successful');
+
+    $parsed_results = id(new GoTestResultParser())
+      ->parseTestResults('subpackage_test.go', $stubbed_results);
+
+    $this->assertEqual(2, count($parsed_results));
+    $this->assertEqual(
+      "Go::Test::package::subpackage::TestFoo",
+      $parsed_results[0]->getName());
+    foreach ($parsed_results as $result) {
+      $this->assertEqual(
+        ArcanistUnitTestResult::RESULT_PASS,
+        $result->getResult());
+    }
+  }
+
+  public function testSingleTestCaseFailure() {
+    $stubbed_results = Filesystem::readFile(
+      dirname(__FILE__).'/testresults/go.single-test-case-failure');
+
+    $parsed_results = id(new GoTestResultParser())
+      ->parseTestResults('subpackage_test.go', $stubbed_results);
+
+    $this->assertEqual(2, count($parsed_results));
+    $this->assertEqual(
+      ArcanistUnitTestResult::RESULT_FAIL,
+      $parsed_results[0]->getResult());
+    $this->assertEqual(
+      ArcanistUnitTestResult::RESULT_PASS,
+      $parsed_results[1]->getResult());
+  }
+
+  public function testMultipleTestCasesSuccessful() {
+    $stubbed_results = Filesystem::readFile(
+      dirname(__FILE__).'/testresults/go.multiple-test-cases-successful');
+
+    $parsed_results = id(new GoTestResultParser())
+      ->parseTestResults('package', $stubbed_results);
+
+    $this->assertEqual(3, count($parsed_results));
+    $this->assertEqual(
+      "Go::Test::package::subpackage1::TestFoo1",
+      $parsed_results[0]->getName());
+    $this->assertEqual(
+      "Go::Test::package::subpackage2::TestFoo2",
+      $parsed_results[2]->getName());
+    foreach ($parsed_results as $result) {
+      $this->assertEqual(
+        ArcanistUnitTestResult::RESULT_PASS,
+        $result->getResult());
+    }
+  }
+
+  public function testMultipleTestCasesFailure() {
+    $stubbed_results = Filesystem::readFile(
+      dirname(__FILE__).'/testresults/go.multiple-test-cases-failure');
+
+    $parsed_results = id(new GoTestResultParser())
+      ->parseTestResults('package', $stubbed_results);
+
+    $this->assertEqual(3, count($parsed_results));
+    $this->assertEqual(
+      "Go::Test::package::subpackage1::TestFoo1",
+      $parsed_results[0]->getName());
+    $this->assertEqual(
+      "Go::Test::package::subpackage2::TestFoo2",
+      $parsed_results[2]->getName());
+    $this->assertEqual(
+      ArcanistUnitTestResult::RESULT_PASS,
+      $parsed_results[0]->getResult());
+    $this->assertEqual(
+      ArcanistUnitTestResult::RESULT_FAIL,
+      $parsed_results[2]->getResult());
+  }
+
+  public function testNonVerboseOutput() {
+    $stubbed_results = Filesystem::readFile(
+      dirname(__FILE__).'/testresults/go.nonverbose');
+
+    $parsed_results = id(new GoTestResultParser())
+      ->parseTestResults('package', $stubbed_results);
+
+    $this->assertEqual(2, count($parsed_results));
+    $this->assertEqual(
+      "Go::TestCase::package::subpackage1",
+      $parsed_results[0]->getName());
+    $this->assertEqual(
+      "Go::TestCase::package::subpackage2",
+      $parsed_results[1]->getName());
+    foreach ($parsed_results as $result) {
+      $this->assertEqual(
+        ArcanistUnitTestResult::RESULT_PASS,
+        $result->getResult());
+    }
+  }
+}

src/unit/engine/__tests__/testresults/go.multiple-test-cases-failure

@@ -0,0 +1,12 @@
+=== RUN TestFoo1
+--- PASS: TestFoo1 (0.03 seconds)
+=== RUN TestBar1
+--- PASS: TestBar1 (0.01 seconds)
+PASS
+ok  	package/subpackage1	0.042s
+=== RUN TestFoo2
+--- FAIL: TestFoo2 (0.02 seconds)
+	subpackage2_test.go:53: got: 2, want: 1
+FAIL
+exit status 1
+FAIL	package/subpackage2	0.020s

src/unit/engine/__tests__/testresults/go.multiple-test-cases-successful

@@ -0,0 +1,10 @@
+=== RUN TestFoo1
+--- PASS: TestFoo1 (0.03 seconds)
+=== RUN TestBar1
+--- PASS: TestBar1 (0.01 seconds)
+PASS
+ok  	package/subpackage1	0.042s
+=== RUN TestFoo2
+--- PASS: TestFoo2 (0.02 seconds)
+PASS
+ok  	package/subpackage2	0.021s

src/unit/engine/__tests__/testresults/go.nonverbose

@@ -0,0 +1,2 @@
+ok  	package/subpackage1	0.042s
+ok  	package/subpackage2	0.021s

src/unit/engine/__tests__/testresults/go.single-test-case-failure

@@ -0,0 +1,8 @@
+=== RUN TestFoo
+--- FAIL: TestFoo (0.03 seconds)
+	subpackage_test.go:53: got: 2, want: 1
+=== RUN TestBar
+--- PASS: TestBar (0.01 seconds)
+FAIL
+exit status 1
+FAIL	package/subpackage	0.042s

src/unit/engine/__tests__/testresults/go.single-test-case-successful

@@ -0,0 +1,6 @@
+=== RUN TestFoo
+--- PASS: TestFoo (0.03 seconds)
+=== RUN TestBar
+--- PASS: TestBar (0.01 seconds)
+PASS
+ok  	package/subpackage	0.042s