revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-26 00:32:42 +01:00
Code Issues Releases Wiki Activity

Add vebose logging to PhutilRepositoryPullDaemon

Browse source 
Summary: Add verbose logging. This logging is activated by setting "phd.verbose" in the config, running "phd debug", or explicitly in scripts/repository/pull.php and scripst/repository/discover.php

Test Plan:
  >>> orbital ~/devtools/phabricator $ ./scripts/repository/discover.php GTEST
  Discovering 'GTEST'...
  <VERB> PhabricatorRepositoryPullLocalDaemon Discovering commits in repository 'GTEST'...
  <VERB> PhabricatorRepositoryPullLocalDaemon Examining branch '()_+abcd$100', at a37bc285a12efa7224fe19f3df54cd90fa2b897a.
  <VERB> PhabricatorRepositoryPullLocalDaemon Skipping, HEAD is known.
  <VERB> PhabricatorRepositoryPullLocalDaemon Examining branch '+abcd$100', at a37bc285a12efa7224fe19f3df54cd90fa2b897a.
  <VERB> PhabricatorRepositoryPullLocalDaemon Skipping, HEAD is known.
  <VERB> PhabricatorRepositoryPullLocalDaemon Examining branch '_+abcd$100', at a37bc285a12efa7224fe19f3df54cd90fa2b897a.
  <VERB> PhabricatorRepositoryPullLocalDaemon Skipping, HEAD is known.
  <VERB> PhabricatorRepositoryPullLocalDaemon Examining branch 'abcd$100', at a37bc285a12efa7224fe19f3df54cd90fa2b897a.
  <VERB> PhabricatorRepositoryPullLocalDaemon Skipping, HEAD is known.
  <VERB> PhabricatorRepositoryPullLocalDaemon Examining branch 'arcpatch', at 774c7737b2d560a291697126bf4513204ccf661a.
  <VERB> PhabricatorRepositoryPullLocalDaemon Skipping, HEAD is known.
  <VERB> PhabricatorRepositoryPullLocalDaemon Examining branch 'arcpatch-1', at dc97539bee07293f95990d71f4638335a2531d69.
  <VERB> PhabricatorRepositoryPullLocalDaemon Skipping, HEAD is known.
  <VERB> PhabricatorRepositoryPullLocalDaemon Examining branch 'arcpatch-2', at 1acfaec313c46dd3caa90448800181fb91b0270f.

Reviewers: jungejason

Reviewed By: jungejason

CC: aran

Differential Revision: https://secure.phabricator.com/D2843
This commit is contained in:
epriestley 2012-06-24 15:06:40 -07:00
parent de46a2550c
commit a705f336a3
3 changed files with 97 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
scripts/repository/discover.php
View file

@ -46,6 +46,9 @@ $repos = PhabricatorRepository::loadAllByPHIDOrCallsign($repo_names);
foreach ($repos as $repo) { foreach ($repos as $repo) {
$callsign = $repo->getCallsign(); $callsign = $repo->getCallsign();
echo "Discovering '{$callsign}'...\n"; echo "Discovering '{$callsign}'...\n";
PhabricatorRepositoryPullLocalDaemon::discoverRepository($repo);
$daemon = new PhabricatorRepositoryPullLocalDaemon(array());
$daemon->setVerbose(true);
$daemon->discoverRepository($repo);
} }
echo "Done.\n"; echo "Done.\n";

5
scripts/repository/pull.php
View file

@ -46,6 +46,9 @@ $repos = PhabricatorRepository::loadAllByPHIDOrCallsign($repo_names);
foreach ($repos as $repo) { foreach ($repos as $repo) {
$callsign = $repo->getCallsign(); $callsign = $repo->getCallsign();
echo "Pulling '{$callsign}'...\n"; echo "Pulling '{$callsign}'...\n";
PhabricatorRepositoryPullLocalDaemon::pullRepository($repo);
$daemon = new PhabricatorRepositoryPullLocalDaemon(array());
$daemon->setVerbose(true);
$daemon->pullRepository($repo);
} }
echo "Done.\n"; echo "Done.\n";

155
src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
View file

@ -45,7 +45,7 @@
final class PhabricatorRepositoryPullLocalDaemon final class PhabricatorRepositoryPullLocalDaemon
extends PhabricatorDaemon { extends PhabricatorDaemon {
private static $commitCache = array(); private $commitCache = array();
/* -( Pulling Repositories )----------------------------------------------- */ /* -( Pulling Repositories )----------------------------------------------- */
@ -127,12 +127,12 @@ final class PhabricatorRepositoryPullLocalDaemon
} }
try { try {
self::pullRepository($repository); $this->pullRepository($repository);
if (!$no_discovery) { if (!$no_discovery) {
// TODO: It would be nice to discover only if we pulled something, // TODO: It would be nice to discover only if we pulled something,
// but this isn't totally trivial. // but this isn't totally trivial.
self::discoverRepository($repository); $this->discoverRepository($repository);
} }
$sleep_for = $repository->getDetail('pull-frequency', $min_sleep); $sleep_for = $repository->getDetail('pull-frequency', $min_sleep);
@ -171,7 +171,7 @@ final class PhabricatorRepositoryPullLocalDaemon
/** /**
* @task pull * @task pull
*/ */
public static function pullRepository(PhabricatorRepository $repository) { public function pullRepository(PhabricatorRepository $repository) {
$vcs = $repository->getVersionControlSystem(); $vcs = $repository->getVersionControlSystem();
$is_svn = ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_SVN); $is_svn = ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_SVN);
@ -202,39 +202,39 @@ final class PhabricatorRepositoryPullLocalDaemon
} }
if ($is_git) { if ($is_git) {
return self::executeGitCreate($repository, $local_path); return $this->executeGitCreate($repository, $local_path);
} else if ($is_hg) { } else if ($is_hg) {
return self::executeHgCreate($repository, $local_path); return $this->executeHgCreate($repository, $local_path);
} }
} else { } else {
if ($is_git) { if ($is_git) {
return self::executeGitUpdate($repository, $local_path); return $this->executeGitUpdate($repository, $local_path);
} else if ($is_hg) { } else if ($is_hg) {
return self::executeHgUpdate($repository, $local_path); return $this->executeHgUpdate($repository, $local_path);
} }
} }
} }
public static function discoverRepository(PhabricatorRepository $repository) { public function discoverRepository(PhabricatorRepository $repository) {
$vcs = $repository->getVersionControlSystem(); $vcs = $repository->getVersionControlSystem();
switch ($vcs) { switch ($vcs) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
return self::executeGitDiscover($repository); return $this->executeGitDiscover($repository);
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
return self::executeSvnDiscover($repository); return $this->executeSvnDiscover($repository);
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
return self::executeHgDiscover($repository); return $this->executeHgDiscover($repository);
default: default:
throw new Exception("Unknown VCS '{$vcs}'!"); throw new Exception("Unknown VCS '{$vcs}'!");
} }
} }
private static function isKnownCommit( private function isKnownCommit(
PhabricatorRepository $repository, PhabricatorRepository $repository,
$target) { $target) {
if (self::getCache($repository, $target)) { if ($this->getCache($repository, $target)) {
return true; return true;
} }
@ -247,15 +247,15 @@ final class PhabricatorRepositoryPullLocalDaemon
return false; return false;
} }
self::setCache($repository, $target); $this->setCache($repository, $target);
while (count(self::$commitCache) > 2048) { while (count($this->commitCache) > 2048) {
array_shift(self::$commitCache); array_shift($this->commitCache);
} }
return true; return true;
} }
private static function isKnownCommitOnAnyAutocloseBranch( private function isKnownCommitOnAnyAutocloseBranch(
PhabricatorRepository $repository, PhabricatorRepository $repository,
$target) { $target) {
@ -280,7 +280,7 @@ final class PhabricatorRepositoryPullLocalDaemon
return false; return false;
} }
private static function recordCommit( private function recordCommit(
PhabricatorRepository $repository, PhabricatorRepository $repository,
$commit_identifier, $commit_identifier,
$epoch) { $epoch) {
@ -300,7 +300,7 @@ final class PhabricatorRepositoryPullLocalDaemon
)); ));
$event->recordEvent(); $event->recordEvent();
self::insertTask($repository, $commit); $this->insertTask($repository, $commit);
queryfx( queryfx(
$repository->establishConnection('w'), $repository->establishConnection('w'),
@ -316,17 +316,17 @@ final class PhabricatorRepositoryPullLocalDaemon
$commit->getID(), $commit->getID(),
$epoch); $epoch);
self::setCache($repository, $commit_identifier); $this->setCache($repository, $commit_identifier);
} catch (AphrontQueryDuplicateKeyException $ex) { } catch (AphrontQueryDuplicateKeyException $ex) {
// Ignore. This can happen because we discover the same new commit // Ignore. This can happen because we discover the same new commit
// more than once when looking at history, or because of races or // more than once when looking at history, or because of races or
// data inconsistency or cosmic radiation; in any case, we're still // data inconsistency or cosmic radiation; in any case, we're still
// in a good state if we ignore the failure. // in a good state if we ignore the failure.
self::setCache($repository, $commit_identifier); $this->setCache($repository, $commit_identifier);
} }
} }
private static function updateCommit( private function updateCommit(
PhabricatorRepository $repository, PhabricatorRepository $repository,
$commit_identifier, $commit_identifier,
$branch) { $branch) {
@ -355,7 +355,7 @@ final class PhabricatorRepositoryPullLocalDaemon
$data->setCommitDetail('seenOnBranches', $branches); $data->setCommitDetail('seenOnBranches', $branches);
$data->save(); $data->save();
self::insertTask( $this->insertTask(
$repository, $repository,
$commit, $commit,
array( array(
@ -363,7 +363,7 @@ final class PhabricatorRepositoryPullLocalDaemon
)); ));
} }
private static function insertTask( private function insertTask(
PhabricatorRepository $repository, PhabricatorRepository $repository,
PhabricatorRepositoryCommit $commit, PhabricatorRepositoryCommit $commit,
$data = array()) { $data = array()) {
@ -392,23 +392,23 @@ final class PhabricatorRepositoryPullLocalDaemon
} }
private static function setCache( private function setCache(
PhabricatorRepository $repository, PhabricatorRepository $repository,
$commit_identifier) { $commit_identifier) {
$key = self::getCacheKey($repository, $commit_identifier); $key = $this->getCacheKey($repository, $commit_identifier);
self::$commitCache[$key] = true; $this->commitCache[$key] = true;
} }
private static function getCache( private function getCache(
PhabricatorRepository $repository, PhabricatorRepository $repository,
$commit_identifier) { $commit_identifier) {
$key = self::getCacheKey($repository, $commit_identifier); $key = $this->getCacheKey($repository, $commit_identifier);
return idx(self::$commitCache, $key, false); return idx($this->commitCache, $key, false);
} }
private static function getCacheKey( private function getCacheKey(
PhabricatorRepository $repository, PhabricatorRepository $repository,
$commit_identifier) { $commit_identifier) {
@ -423,7 +423,7 @@ final class PhabricatorRepositoryPullLocalDaemon
/** /**
* @task git * @task git
*/ */
private static function executeGitCreate( private function executeGitCreate(
PhabricatorRepository $repository, PhabricatorRepository $repository,
$path) { $path) {
@ -437,7 +437,7 @@ final class PhabricatorRepositoryPullLocalDaemon
/** /**
* @task git * @task git
*/ */
private static function executeGitUpdate( private function executeGitUpdate(
PhabricatorRepository $repository, PhabricatorRepository $repository,
$path) { $path) {
@ -502,7 +502,7 @@ final class PhabricatorRepositoryPullLocalDaemon
/** /**
* @task git * @task git
*/ */
private static function executeGitDiscover( private function executeGitDiscover(
PhabricatorRepository $repository) { PhabricatorRepository $repository) {
list($remotes) = $repository->execxLocalCommand( list($remotes) = $repository->execxLocalCommand(
@ -514,7 +514,7 @@ final class PhabricatorRepositoryPullLocalDaemon
"Expected 'Fetch URL' in 'git remote show -n origin'."); "Expected 'Fetch URL' in 'git remote show -n origin'.");
} }
self::executeGitverifySameOrigin( self::executeGitVerifySameOrigin(
$matches[1], $matches[1],
$repository->getRemoteURI(), $repository->getRemoteURI(),
$repository->getLocalPath()); $repository->getLocalPath());
@ -526,19 +526,27 @@ final class PhabricatorRepositoryPullLocalDaemon
$stdout, $stdout,
$only_this_remote = DiffusionBranchInformation::DEFAULT_GIT_REMOTE); $only_this_remote = DiffusionBranchInformation::DEFAULT_GIT_REMOTE);
$callsign = $repository->getCallsign();
$tracked_something = false; $tracked_something = false;
$this->log("Discovering commits in repository '{$callsign}'...");
foreach ($branches as $name => $commit) { foreach ($branches as $name => $commit) {
$this->log("Examining branch '{$name}', at {$commit}.");
if (!$repository->shouldTrackBranch($name)) { if (!$repository->shouldTrackBranch($name)) {
$this->log("Skipping, branch is untracked.");
continue; continue;
} }
$tracked_something = true; $tracked_something = true;
if (self::isKnownCommit($repository, $commit)) { if ($this->isKnownCommit($repository, $commit)) {
$this->log("Skipping, HEAD is known.");
continue; continue;
} else {
self::executeGitDiscoverCommit($repository, $commit);
} }
$this->log("Looking for new commits.");
$this->executeGitDiscoverCommit($repository, $commit);
} }
if (!$tracked_something) { if (!$tracked_something) {
@ -549,20 +557,27 @@ final class PhabricatorRepositoryPullLocalDaemon
"Verify that your branch filtering settings are correct."); "Verify that your branch filtering settings are correct.");
} }
$this->log("Discovering commits on autoclose branches...");
foreach ($branches as $name => $commit) { foreach ($branches as $name => $commit) {
$this->log("Examining branch '{$name}', at {$commit}'.");
if (!$repository->shouldTrackBranch($name)) { if (!$repository->shouldTrackBranch($name)) {
$this->log("Skipping, branch is untracked.");
continue; continue;
} }
if (!$repository->shouldAutocloseBranch($name)) { if (!$repository->shouldAutocloseBranch($name)) {
$this->log("Skipping, branch is not autoclose.");
continue; continue;
} }
if (self::isKnownCommitOnAnyAutocloseBranch($repository, $commit)) { if ($this->isKnownCommitOnAnyAutocloseBranch($repository, $commit)) {
$this->log("Skipping, commit is known on an autoclose branch.");
continue; continue;
} }
self::executeGitDiscoverCommit($repository, $commit, $name); $this->log("Looking for new autoclose commits.");
$this->executeGitDiscoverCommit($repository, $commit, $name);
} }
} }
@ -570,7 +585,7 @@ final class PhabricatorRepositoryPullLocalDaemon
/** /**
* @task git * @task git
*/ */
private static function executeGitDiscoverCommit( private function executeGitDiscoverCommit(
PhabricatorRepository $repository, PhabricatorRepository $repository,
$commit, $commit,
$branch = null) { $branch = null) {
@ -596,13 +611,14 @@ final class PhabricatorRepositoryPullLocalDaemon
} }
$seen_parent[$parent] = true; $seen_parent[$parent] = true;
if ($branch !== null) { if ($branch !== null) {
$known = self::isKnownCommitOnAnyAutocloseBranch( $known = $this->isKnownCommitOnAnyAutocloseBranch(
$repository, $repository,
$parent); $parent);
} else { } else {
$known = self::isKnownCommit($repository, $parent); $known = $this->isKnownCommit($repository, $parent);
} }
if (!$known) { if (!$known) {
$this->log("Discovered commit '{$parent}'.");
$discover[] = $parent; $discover[] = $parent;
$insert[] = $parent; $insert[] = $parent;
} }
@ -612,6 +628,13 @@ final class PhabricatorRepositoryPullLocalDaemon
} }
} }
$n = count($insert);
if ($branch !== null) {
$this->log("Found {$n} new autoclose commits on branch '{$branch}'.");
} else {
$this->log("Found {$n} new commits.");
}
while (true) { while (true) {
$target = array_pop($insert); $target = array_pop($insert);
list($epoch) = $repository->execxLocalCommand( list($epoch) = $repository->execxLocalCommand(
@ -620,9 +643,9 @@ final class PhabricatorRepositoryPullLocalDaemon
$epoch = trim($epoch); $epoch = trim($epoch);
if ($branch !== null) { if ($branch !== null) {
self::updateCommit($repository, $target, $branch); $this->updateCommit($repository, $target, $branch);
} else { } else {
self::recordCommit($repository, $target, $epoch); $this->recordCommit($repository, $target, $epoch);
} }
if (empty($insert)) { if (empty($insert)) {
@ -674,7 +697,7 @@ final class PhabricatorRepositoryPullLocalDaemon
/** /**
* @task hg * @task hg
*/ */
private static function executeHgCreate( private function executeHgCreate(
PhabricatorRepository $repository, PhabricatorRepository $repository,
$path) { $path) {
@ -688,7 +711,7 @@ final class PhabricatorRepositoryPullLocalDaemon
/** /**
* @task hg * @task hg
*/ */
private static function executeHgUpdate( private function executeHgUpdate(
PhabricatorRepository $repository, PhabricatorRepository $repository,
$path) { $path) {
@ -725,7 +748,7 @@ final class PhabricatorRepositoryPullLocalDaemon
} }
} }
private static function executeHgDiscover(PhabricatorRepository $repository) { private function executeHgDiscover(PhabricatorRepository $repository) {
// NOTE: "--debug" gives us 40-character hashes. // NOTE: "--debug" gives us 40-character hashes.
list($stdout) = $repository->execxLocalCommand('--debug branches'); list($stdout) = $repository->execxLocalCommand('--debug branches');
@ -733,10 +756,10 @@ final class PhabricatorRepositoryPullLocalDaemon
$got_something = false; $got_something = false;
foreach ($branches as $name => $branch) { foreach ($branches as $name => $branch) {
$commit = $branch['rev']; $commit = $branch['rev'];
if (self::isKnownCommit($repository, $commit)) { if ($this->isKnownCommit($repository, $commit)) {
continue; continue;
} else { } else {
self::executeHgDiscoverCommit($repository, $commit); $this->executeHgDiscoverCommit($repository, $commit);
$got_something = true; $got_something = true;
} }
} }
@ -744,7 +767,7 @@ final class PhabricatorRepositoryPullLocalDaemon
return $got_something; return $got_something;
} }
private static function executeHgDiscoverCommit( private function executeHgDiscoverCommit(
PhabricatorRepository $repository, PhabricatorRepository $repository,
$commit) { $commit) {
@ -767,7 +790,7 @@ final class PhabricatorRepositoryPullLocalDaemon
continue; continue;
} }
$seen_parent[$parent] = true; $seen_parent[$parent] = true;
if (!self::isKnownCommit($repository, $parent)) { if (!$this->isKnownCommit($repository, $parent)) {
$discover[] = $parent; $discover[] = $parent;
$insert[] = $parent; $insert[] = $parent;
} }
@ -776,7 +799,7 @@ final class PhabricatorRepositoryPullLocalDaemon
foreach ($insert as $target) { foreach ($insert as $target) {
$epoch = $stream->getCommitDate($target); $epoch = $stream->getCommitDate($target);
self::recordCommit($repository, $target, $epoch); $this->recordCommit($repository, $target, $epoch);
} }
} }
@ -784,33 +807,33 @@ final class PhabricatorRepositoryPullLocalDaemon
/* -( Subversion Implementation )------------------------------------------ */ /* -( Subversion Implementation )------------------------------------------ */
private static function executeSvnDiscover( private function executeSvnDiscover(
PhabricatorRepository $repository) { PhabricatorRepository $repository) {
$uri = self::executeSvnGetBaseSVNLogURI($repository); $uri = $this->executeSvnGetBaseSVNLogURI($repository);
list($xml) = $repository->execxRemoteCommand( list($xml) = $repository->execxRemoteCommand(
'log --xml --quiet --limit 1 %s@HEAD', 'log --xml --quiet --limit 1 %s@HEAD',
$uri); $uri);
$results = self::executeSvnParseLogXML($xml); $results = $this->executeSvnParseLogXML($xml);
$commit = head_key($results); $commit = head_key($results);
$epoch = head($results); $epoch = head($results);
if (self::isKnownCommit($repository, $commit)) { if ($this->isKnownCommit($repository, $commit)) {
return false; return false;
} }
self::executeSvnDiscoverCommit($repository, $commit, $epoch); $this->executeSvnDiscoverCommit($repository, $commit, $epoch);
return true; return true;
} }
private static function executeSvnDiscoverCommit( private function executeSvnDiscoverCommit(
PhabricatorRepository $repository, PhabricatorRepository $repository,
$commit, $commit,
$epoch) { $epoch) {
$uri = self::executeSvnGetBaseSVNLogURI($repository); $uri = $this->executeSvnGetBaseSVNLogURI($repository);
$discover = array( $discover = array(
$commit => $epoch, $commit => $epoch,
@ -819,7 +842,7 @@ final class PhabricatorRepositoryPullLocalDaemon
$limit = 1; $limit = 1;
while ($upper_bound > 1 && while ($upper_bound > 1 &&
!self::isKnownCommit($repository, $upper_bound)) { !$this->isKnownCommit($repository, $upper_bound)) {
// Find all the unknown commits on this path. Note that we permit // Find all the unknown commits on this path. Note that we permit
// importing an SVN subdirectory rather than the entire repository, so // importing an SVN subdirectory rather than the entire repository, so
// commits may be nonsequential. // commits may be nonsequential.
@ -837,7 +860,7 @@ final class PhabricatorRepositoryPullLocalDaemon
throw new Exception("svn log error #{$err}: {$stderr}"); throw new Exception("svn log error #{$err}: {$stderr}");
} }
} }
$discover += self::executeSvnParseLogXML($xml); $discover += $this->executeSvnParseLogXML($xml);
$upper_bound = min(array_keys($discover)); $upper_bound = min(array_keys($discover));
@ -856,11 +879,11 @@ final class PhabricatorRepositoryPullLocalDaemon
ksort($discover); ksort($discover);
foreach ($discover as $commit => $epoch) { foreach ($discover as $commit => $epoch) {
self::recordCommit($repository, $commit, $epoch); $this->recordCommit($repository, $commit, $epoch);
} }
} }
private static function executeSvnParseLogXML($xml) { private function executeSvnParseLogXML($xml) {
$xml = phutil_utf8ize($xml); $xml = phutil_utf8ize($xml);
$result = array(); $result = array();
@ -876,7 +899,7 @@ final class PhabricatorRepositoryPullLocalDaemon
} }
private static function executeSvnGetBaseSVNLogURI( private function executeSvnGetBaseSVNLogURI(
PhabricatorRepository $repository) { PhabricatorRepository $repository) {
$uri = $repository->getDetail('remote-uri'); $uri = $repository->getDetail('remote-uri');