1
0
Fork 0
mirror of https://we.phorge.it/source/arcanist.git synced 2024-11-22 14:52:40 +01:00

Added base test result parser class. Implemented Go test result parser based on Go's built in test utility output.

Test Plan: Ran PhpunitTestEngine unit test and used both test result parsers to generate test results.

Reviewers: epriestley

Reviewed By: epriestley

CC: aran, Korvin, aurelijus

Differential Revision: https://secure.phabricator.com/D4676
This commit is contained in:
indiefan 2013-01-27 17:16:50 -08:00 committed by epriestley
parent c49997a6e8
commit ce0a491bcd
11 changed files with 394 additions and 78 deletions

View file

@ -18,6 +18,7 @@ phutil_register_library_map(array(
'ArcanistArcanistLinterTestCase' => 'lint/linter/__tests__/ArcanistArcanistLinterTestCase.php',
'ArcanistBaseCommitParser' => 'parser/ArcanistBaseCommitParser.php',
'ArcanistBaseCommitParserTestCase' => 'parser/__tests__/ArcanistBaseCommitParserTestCase.php',
'ArcanistBaseTestResultParser' => 'unit/engine/ArcanistBaseTestResultParser.php',
'ArcanistBaseUnitTestEngine' => 'unit/engine/ArcanistBaseUnitTestEngine.php',
'ArcanistBaseWorkflow' => 'workflow/ArcanistBaseWorkflow.php',
'ArcanistBranchWorkflow' => 'workflow/ArcanistBranchWorkflow.php',
@ -143,6 +144,8 @@ phutil_register_library_map(array(
'ArcanistXHPASTLinterTestCase' => 'lint/linter/__tests__/ArcanistXHPASTLinterTestCase.php',
'ComprehensiveLintEngine' => 'lint/engine/ComprehensiveLintEngine.php',
'ExampleLintEngine' => 'lint/engine/ExampleLintEngine.php',
'GoTestResultParser' => 'unit/engine/GoTestResultParser.php',
'GoTestResultParserTestCase' => 'unit/engine/__tests__/GoTestResultParserTestCase.php',
'NoseTestEngine' => 'unit/engine/NoseTestEngine.php',
'PHPUnitTestEngineTestCase' => 'unit/engine/__tests__/PHPUnitTestEngineTestCase.php',
'PhpunitResultParser' => 'unit/engine/PhpunitResultParser.php',
@ -259,8 +262,11 @@ phutil_register_library_map(array(
'ArcanistXHPASTLinterTestCase' => 'ArcanistArcanistLinterTestCase',
'ComprehensiveLintEngine' => 'ArcanistLintEngine',
'ExampleLintEngine' => 'ArcanistLintEngine',
'GoTestResultParser' => 'ArcanistBaseTestResultParser',
'GoTestResultParserTestCase' => 'ArcanistTestCase',
'NoseTestEngine' => 'ArcanistBaseUnitTestEngine',
'PHPUnitTestEngineTestCase' => 'ArcanistTestCase',
'PhpunitResultParser' => 'ArcanistBaseTestResultParser',
'PhpunitTestEngine' => 'ArcanistBaseUnitTestEngine',
'PhutilLintEngine' => 'ArcanistLintEngine',
'PhutilUnitTestEngine' => 'ArcanistBaseUnitTestEngine',

View file

@ -0,0 +1,46 @@
<?php
/**
* Abstract Base class for test result parsers
*/
abstract class ArcanistBaseTestResultParser {
protected $enableCoverage;
protected $projectRoot;
protected $coverateFile;
public function setEnableCoverage($enable_coverage) {
$this->enableCoverage = $enable_coverage;
return $this;
}
public function setProjectRoot($project_root) {
$this->projectRoot = $project_root;
return $this;
}
public function setCoverageFile($coverage_file) {
$this->coverageFile = $coverage_file;
return $this;
}
public function setAffectedTests($affected_tests) {
$this->affectedTests = $affected_tests;
return $this;
}
/**
* Parse test results from provided input and return an array
* of ArcanistUnitTestResult
*
* @param string $path Path to test
* @param string $test_results String containing test results
*
* @return array ArcanistUnitTestResult
*/
abstract public function parseTestResults($path, $test_results);
}

View file

@ -0,0 +1,140 @@
<?php
/**
* Go Test Result Parsing utility
*
* Intended to enable custom unit engines derived
* from Go's built-in test utility to reuse
* common business logic related to parsing
* Go test results.
*
* (To generate test output, run something like:
* `go test -v`)
*/
final class GoTestResultParser extends ArcanistBaseTestResultParser {
/**
* Parse test results from Go test report
* (e.g. `go test -v`)
*
* @param string $path Path to test
* @param string $test_results String containing Go test output
*
* @return array
*/
public function parseTestResults($path, $test_results) {
$test_results = explode("\n", $test_results);
$results = array();
// We'll get our full test case name at the end and add it back in
$test_case_name = "";
// Temp store for test case results (in case we run multiple test cases)
$test_case_results = array();
foreach ($test_results as $i => $line) {
if (strncmp($line, "--- PASS", 8) === 0) {
// We have a passing test
$meta = array();
preg_match(
'/^--- PASS: (?P<test_name>.+) \((?P<time>.+) seconds\).*/',
$line,
$meta);
$result = new ArcanistUnitTestResult();
// For now set name without test case, we'll add it later
$result->setName($meta['test_name']);
$result->setResult(ArcanistUnitTestResult::RESULT_PASS);
$result->setDuration($meta['time']);
$test_case_results[] = $result;
continue;
}
if (strncmp($line, "--- FAIL", 8) === 0) {
// We have a failing test
$reason = trim($test_results[$i + 1]);
$meta = array();
preg_match(
'/^--- FAIL: (?P<test_name>.+) \((?P<time>.+) seconds\).*/',
$line,
$meta);
$result = new ArcanistUnitTestResult();
$result->setName($meta['test_name']);
$result->setResult(ArcanistUnitTestResult::RESULT_FAIL);
$result->setDuration($meta['time']);
$result->setUserData($reason."\n");
$test_case_results[] = $result;
continue;
}
if (strncmp($line, "ok", 2) === 0) {
$meta = array();
preg_match(
'/^ok[\s\t]+(?P<test_name>\w.*)[\s\t]+(?P<time>.*)s.*/',
$line,
$meta);
$test_case_name = str_replace("/", "::", $meta['test_name']);
// Our test case passed
// check to make sure we were in verbose (-v) mode
if (empty($test_case_results)) {
// We weren't in verbose mode
// create one successful result for the whole test case
$test_name = "Go::TestCase::".$test_case_name;
$result = new ArcanistUnitTestResult();
$result->setName($test_name);
$result->setResult(ArcanistUnitTestResult::RESULT_PASS);
$result->setDuration($meta['time']);
$results[] = $result;
} else {
$test_case_results = $this->fixNames(
$test_case_results,
$test_case_name);
$results = array_merge($results, $test_case_results);
$test_case_results = array();
}
continue;
}
if (strncmp($line, "FAIL\t", 5) === 0) {
$meta = array();
preg_match(
'/^FAIL[\s\t]+(?P<test_name>\w.*)[\s\t]+.*/',
$line,
$meta);
$test_case_name = str_replace("/", "::", $meta['test_name']);
$test_case_results = $this->fixNames(
$test_case_results,
$test_case_name);
$results = array_merge($results, $test_case_results);
$test_case_results = array();
continue;
}
}
return $results;
}
private function fixNames($test_case_results, $test_case_name) {
foreach ($test_case_results as &$result) {
$test_name = $result->getName();
$result->setName("Go::Test::".$test_case_name."::".$test_name);
}
return $test_case_results;
}
}

View file

@ -11,105 +11,79 @@
* engine, see PhpunitTestEngine.
*
*/
final class PhpunitResultParser {
private $enableCoverage;
private $projectRoot;
public function setEnableCoverage($enable_coverage) {
$this->enableCoverage = $enable_coverage;
return $this;
}
public function setProjectRoot($project_root) {
$this->projectRoot = $project_root;
return $this;
}
final class PhpunitResultParser extends ArcanistBaseTestResultParser {
/**
* Parse test results from phpunit json report
*
* @param string $path Path to test
* @param string $json_path Path to phpunit json report
* @param string $clover_tmp Path to phpunit clover report
* @param array $affected_tests Array of the tests affected by this run
* @param bool $enable_coverage Option to enable coverage in results
* @param string $test_results String containing phpunit json report
*
* @return array
*/
public function parseTestResults(
$path,
$json_tmp,
$clover_tmp,
$affected_tests) {
public function parseTestResults($path, $test_results) {
$test_results = Filesystem::readFile($json_tmp);
$report = $this->getJsonReport($test_results);
$report = $this->getJsonReport($json_tmp);
// coverage is for all testcases in the executed $path
$coverage = array();
if ($this->enableCoverage !== false) {
$coverage = $this->readCoverage();
}
// coverage is for all testcases in the executed $path
$coverage = array();
if ($this->enableCoverage !== false) {
$coverage = $this->readCoverage($clover_tmp, $affected_tests);
$results = array();
foreach ($report as $event) {
if ('test' != $event->event) {
continue;
}
$results = array();
foreach ($report as $event) {
if ('test' != $event->event) {
continue;
$status = ArcanistUnitTestResult::RESULT_PASS;
$user_data = '';
if ('fail' == $event->status) {
$status = ArcanistUnitTestResult::RESULT_FAIL;
$user_data .= $event->message . "\n";
foreach ($event->trace as $trace) {
$user_data .= sprintf("\n%s:%s", $trace->file, $trace->line);
}
$status = ArcanistUnitTestResult::RESULT_PASS;
$user_data = '';
if ('fail' == $event->status) {
$status = ArcanistUnitTestResult::RESULT_FAIL;
$user_data .= $event->message . "\n";
} else if ('error' == $event->status) {
if (strpos($event->message, 'Skipped Test') !== false) {
$status = ArcanistUnitTestResult::RESULT_SKIP;
$user_data .= $event->message;
} else if (strpos($event->message, 'Incomplete Test') !== false) {
$status = ArcanistUnitTestResult::RESULT_SKIP;
$user_data .= $event->message;
} else {
$status = ArcanistUnitTestResult::RESULT_BROKEN;
$user_data .= $event->message;
foreach ($event->trace as $trace) {
$user_data .= sprintf("\n%s:%s", $trace->file, $trace->line);
}
} else if ('error' == $event->status) {
if (strpos($event->message, 'Skipped Test') !== false) {
$status = ArcanistUnitTestResult::RESULT_SKIP;
$user_data .= $event->message;
} else if (strpos($event->message, 'Incomplete Test') !== false) {
$status = ArcanistUnitTestResult::RESULT_SKIP;
$user_data .= $event->message;
} else {
$status = ArcanistUnitTestResult::RESULT_BROKEN;
$user_data .= $event->message;
foreach ($event->trace as $trace) {
$user_data .= sprintf("\n%s:%s", $trace->file, $trace->line);
}
}
}
$name = preg_replace('/ \(.*\)/', '', $event->test);
$result = new ArcanistUnitTestResult();
$result->setName($name);
$result->setResult($status);
$result->setDuration($event->time);
$result->setCoverage($coverage);
$result->setUserData($user_data);
$results[] = $result;
}
return $results;
$name = preg_replace('/ \(.*\)/', '', $event->test);
$result = new ArcanistUnitTestResult();
$result->setName($name);
$result->setResult($status);
$result->setDuration($event->time);
$result->setCoverage($coverage);
$result->setUserData($user_data);
$results[] = $result;
}
return $results;
}
/**
* Read the coverage from phpunit generated clover report
*
* @param string $path Path to report
*
* @return array
*/
private function readCoverage($path, $affected_tests) {
$test_results = Filesystem::readFile($path);
private function readCoverage() {
$test_results = Filesystem::readFile($this->coverageFile);
if (empty($test_results)) {
throw new Exception('Clover coverage XML report file is empty, '
. 'it probably means that phpunit failed to run tests. '
@ -127,10 +101,10 @@ final class PhpunitResultParser {
foreach ($files as $file) {
$class_path = $file->getAttribute('name');
if (empty($affected_tests[$class_path])) {
if (empty($this->affectedTests[$class_path])) {
continue;
}
$test_path = $affected_tests[$file->getAttribute('name')];
$test_path = $this->affectedTests[$file->getAttribute('name')];
// get total line count in file
$line_count = count(file($class_path));
@ -172,12 +146,11 @@ final class PhpunitResultParser {
* We need this non-sense to make json generated by phpunit
* valid.
*
* @param string $json_tmp Path to JSON report
* @param string $json String containing JSON report
*
* @return array JSON decoded array
*/
private function getJsonReport($json_tmp) {
$json = Filesystem::readFile($json_tmp);
private function getJsonReport($json) {
if (empty($json)) {
throw new Exception('JSON report file is empty, '

View file

@ -100,10 +100,13 @@ final class PhpunitTestEngine extends ArcanistBaseUnitTestEngine {
* @return array
*/
private function parseTestResults($path, $json_tmp, $clover_tmp) {
$test_results = Filesystem::readFile($json_tmp);
return id(new PhpunitResultParser())
->setEnableCoverage($this->getEnableCoverage())
->setProjectRoot($this->projectRoot)
->parseTestResults($path, $json_tmp, $clover_tmp, $this->affectedTests);
->setCoverageFile($clover_tmp)
->setAffectedTests($this->affectedTests)
->parseTestResults($path, $test_results);
}

View file

@ -0,0 +1,110 @@
<?php
/**
* Test for @{class:GoTestResultParser}.
*
* (putting tests in your tests so you can test
* while you test)
*
* @group testcase
*/
final class GoTestResultParserTestCase extends ArcanistTestCase {
public function testSingleTestCaseSuccessful() {
$stubbed_results = Filesystem::readFile(
dirname(__FILE__).'/testresults/go.single-test-case-successful');
$parsed_results = id(new GoTestResultParser())
->parseTestResults('subpackage_test.go', $stubbed_results);
$this->assertEqual(2, count($parsed_results));
$this->assertEqual(
"Go::Test::package::subpackage::TestFoo",
$parsed_results[0]->getName());
foreach ($parsed_results as $result) {
$this->assertEqual(
ArcanistUnitTestResult::RESULT_PASS,
$result->getResult());
}
}
public function testSingleTestCaseFailure() {
$stubbed_results = Filesystem::readFile(
dirname(__FILE__).'/testresults/go.single-test-case-failure');
$parsed_results = id(new GoTestResultParser())
->parseTestResults('subpackage_test.go', $stubbed_results);
$this->assertEqual(2, count($parsed_results));
$this->assertEqual(
ArcanistUnitTestResult::RESULT_FAIL,
$parsed_results[0]->getResult());
$this->assertEqual(
ArcanistUnitTestResult::RESULT_PASS,
$parsed_results[1]->getResult());
}
public function testMultipleTestCasesSuccessful() {
$stubbed_results = Filesystem::readFile(
dirname(__FILE__).'/testresults/go.multiple-test-cases-successful');
$parsed_results = id(new GoTestResultParser())
->parseTestResults('package', $stubbed_results);
$this->assertEqual(3, count($parsed_results));
$this->assertEqual(
"Go::Test::package::subpackage1::TestFoo1",
$parsed_results[0]->getName());
$this->assertEqual(
"Go::Test::package::subpackage2::TestFoo2",
$parsed_results[2]->getName());
foreach ($parsed_results as $result) {
$this->assertEqual(
ArcanistUnitTestResult::RESULT_PASS,
$result->getResult());
}
}
public function testMultipleTestCasesFailure() {
$stubbed_results = Filesystem::readFile(
dirname(__FILE__).'/testresults/go.multiple-test-cases-failure');
$parsed_results = id(new GoTestResultParser())
->parseTestResults('package', $stubbed_results);
$this->assertEqual(3, count($parsed_results));
$this->assertEqual(
"Go::Test::package::subpackage1::TestFoo1",
$parsed_results[0]->getName());
$this->assertEqual(
"Go::Test::package::subpackage2::TestFoo2",
$parsed_results[2]->getName());
$this->assertEqual(
ArcanistUnitTestResult::RESULT_PASS,
$parsed_results[0]->getResult());
$this->assertEqual(
ArcanistUnitTestResult::RESULT_FAIL,
$parsed_results[2]->getResult());
}
public function testNonVerboseOutput() {
$stubbed_results = Filesystem::readFile(
dirname(__FILE__).'/testresults/go.nonverbose');
$parsed_results = id(new GoTestResultParser())
->parseTestResults('package', $stubbed_results);
$this->assertEqual(2, count($parsed_results));
$this->assertEqual(
"Go::TestCase::package::subpackage1",
$parsed_results[0]->getName());
$this->assertEqual(
"Go::TestCase::package::subpackage2",
$parsed_results[1]->getName());
foreach ($parsed_results as $result) {
$this->assertEqual(
ArcanistUnitTestResult::RESULT_PASS,
$result->getResult());
}
}
}

View file

@ -0,0 +1,12 @@
=== RUN TestFoo1
--- PASS: TestFoo1 (0.03 seconds)
=== RUN TestBar1
--- PASS: TestBar1 (0.01 seconds)
PASS
ok package/subpackage1 0.042s
=== RUN TestFoo2
--- FAIL: TestFoo2 (0.02 seconds)
subpackage2_test.go:53: got: 2, want: 1
FAIL
exit status 1
FAIL package/subpackage2 0.020s

View file

@ -0,0 +1,10 @@
=== RUN TestFoo1
--- PASS: TestFoo1 (0.03 seconds)
=== RUN TestBar1
--- PASS: TestBar1 (0.01 seconds)
PASS
ok package/subpackage1 0.042s
=== RUN TestFoo2
--- PASS: TestFoo2 (0.02 seconds)
PASS
ok package/subpackage2 0.021s

View file

@ -0,0 +1,2 @@
ok package/subpackage1 0.042s
ok package/subpackage2 0.021s

View file

@ -0,0 +1,8 @@
=== RUN TestFoo
--- FAIL: TestFoo (0.03 seconds)
subpackage_test.go:53: got: 2, want: 1
=== RUN TestBar
--- PASS: TestBar (0.01 seconds)
FAIL
exit status 1
FAIL package/subpackage 0.042s

View file

@ -0,0 +1,6 @@
=== RUN TestFoo
--- PASS: TestFoo (0.03 seconds)
=== RUN TestBar
--- PASS: TestBar (0.01 seconds)
PASS
ok package/subpackage 0.042s