revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-22 06:42:42 +01:00
Code Issues Releases Wiki Activity

Use one daemon to discover commits in all repositories, not one per repository

Browse source 
Summary:
See D2418. This merges the commit discovery daemon into the same single daemon, and applies all the same rules to it.

There are relatively few implementation changes, but a few things did change:

  - I simplified/improved Mercurial importing, by finding full branch tip hashes with "--debug branches" and using "parents --template {node}" so we don't need to do separate "--debug id" calls.
  - Added a new "--not" flag to exclude repositories, since I switched to real arg parsing anyway.
  - I removed a web UI notification that you need to restart the daemons, this is no longer true.
  - I added a web UI notification that no pull daemon is running on the machine.

NOTE: @makinde, this doesn't change anything from your perspective, but it something breaks this is the likely cause.

This implicitly resolves T792, because discovery no longer runs before pulling.

Test Plan:

  - Swapped databases to a fresh install.
  - Ran "pulllocal" in debug mode. Verified it correctly does nothing (fixed a minor issue with min() on empty array).
  - Added an SVN repository. Verified it cloned and discovered correctly.
  - Added a Mercurial repository. Verified it cloned and discovered correctly.
  - Added a Git repository. Verified it cloned and discovered correctly.
  - Ran with arguments to verify behaviors: "--not MTEST --not STEST", "P --no-discovery", "P".

Reviewers: btrahan, csilvers, Makinde

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T792

Differential Revision: https://secure.phabricator.com/D2430
This commit is contained in:
epriestley 2012-05-08 12:53:41 -07:00
parent 679f778235
commit d2b01aead0
24 changed files with 637 additions and 748 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
scripts/daemon/phabricator_daemon_launcher.php
View file

@ -52,13 +52,20 @@ switch (isset($argv[1]) ? $argv[1] : 'help') {
$need_launch = phd_load_tracked_repositories();
if (!$need_launch) {
echo "There are no repositories with tracking enabled.\n";
exit(0);
exit(1);
}
will_launch($control);
echo "Launching PullLocal daemon in readonly mode...\n";
$control->launchDaemon(
'PhabricatorRepositoryPullLocalDaemon',
array());
array(
'--no-discovery',
));
echo "Done.\n";
break;
case 'repository-launch-master':
@ -66,55 +73,24 @@ switch (isset($argv[1]) ? $argv[1] : 'help') {
if (!$need_launch) {
echo "There are no repositories with tracking enabled.\n";
exit(1);
} else {
}
will_launch($control);
echo "Launching PullLocal daemon in master mode...\n";
$control->launchDaemon(
'PhabricatorRepositoryPullLocalDaemon',
array());
foreach ($need_launch as $repository) {
$name = $repository->getName();
$callsign = $repository->getCallsign();
$desc = "'{$name}' ({$callsign})";
$phid = $repository->getPHID();
switch ($repository->getVersionControlSystem()) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
echo "Launching discovery daemon on the {$desc} repository...\n";
$control->launchDaemon(
'PhabricatorRepositoryGitCommitDiscoveryDaemon',
array(
$phid,
));
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
echo "Launching discovery daemon on the {$desc} repository...\n";
$control->launchDaemon(
'PhabricatorRepositorySvnCommitDiscoveryDaemon',
array(
$phid,
));
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
echo "Launching discovery daemon on the {$desc} repository...\n";
$control->launchDaemon(
'PhabricatorRepositoryMercurialCommitDiscoveryDaemon',
array(
$phid,
));
break;
}
}
echo "Launching CommitTask daemon...\n";
$control->launchDaemon(
'PhabricatorRepositoryCommitTaskDaemon',
array());
echo "NOTE: Make sure you run some taskmaster daemons too, e.g. ".
"with 'phd launch 4 taskmaster'.\n";
echo "Done.\n";
}
break;
case 'launch':

51
scripts/repository/discover.php Executable file
View file

@ -0,0 +1,51 @@
#!/usr/bin/env php
<?php
/*
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
$root = dirname(dirname(dirname(__FILE__)));
require_once $root.'/scripts/__init_script__.php';
$args = new PhutilArgumentParser($argv);
$args->setTagline('manually discover working copies');
$args->setSynopsis(<<<EOHELP
**discover.php** [__options__] __repository-callsign-or-phid ...__
Manually discover commits in working copies for the named repositories.
EOHELP
);
$args->parseStandardArguments();
$args->parse(
array(
array(
'name' => 'repositories',
'wildcard' => true,
),
));
$repo_names = $args->getArg('repositories');
if (!$repo_names) {
echo "Specify one or more repositories to pull, by callsign or PHID.\n";
exit(1);
}
$repos = PhabricatorRepository::loadAllByPHIDOrCallsign($repo_names);
foreach ($repos as $repo) {
$callsign = $repo->getCallsign();
echo "Discovering '{$callsign}'...\n";
PhabricatorRepositoryPullLocalDaemon::discoverRepository($repo);
}
echo "Done.\n";

16
src/__phutil_library_map__.php
View file

@ -840,7 +840,6 @@ phutil_register_library_map(array(
'PhabricatorRepositoryCommit' => 'applications/repository/storage/commit',
'PhabricatorRepositoryCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/base',
'PhabricatorRepositoryCommitData' => 'applications/repository/storage/commitdata',
'PhabricatorRepositoryCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/base',
'PhabricatorRepositoryCommitHeraldWorker' => 'applications/repository/worker/herald',
'PhabricatorRepositoryCommitMessageDetailParser' => 'applications/repository/parser/base',
'PhabricatorRepositoryCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/base',
@ -850,22 +849,18 @@ phutil_register_library_map(array(
'PhabricatorRepositoryController' => 'applications/repository/controller/base',
'PhabricatorRepositoryCreateController' => 'applications/repository/controller/create',
'PhabricatorRepositoryDAO' => 'applications/repository/storage/base',
'PhabricatorRepositoryDaemon' => 'applications/repository/daemon/base',
'PhabricatorRepositoryDefaultCommitMessageDetailParser' => 'applications/repository/parser/default',
'PhabricatorRepositoryDeleteController' => 'applications/repository/controller/delete',
'PhabricatorRepositoryEditController' => 'applications/repository/controller/edit',
'PhabricatorRepositoryGitCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/git',
'PhabricatorRepositoryGitCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/git',
'PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase' => 'applications/repository/daemon/commitdiscovery/git/__tests__',
'PhabricatorRepositoryGitCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/git',
'PhabricatorRepositoryListController' => 'applications/repository/controller/list',
'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/mercurial',
'PhabricatorRepositoryMercurialCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/mercurial',
'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/mercurial',
'PhabricatorRepositoryPullLocalDaemon' => 'applications/repository/daemon/pulllocal',
'PhabricatorRepositoryPullLocalDaemonTestCase' => 'applications/repository/daemon/pulllocal/__tests__',
'PhabricatorRepositoryShortcut' => 'applications/repository/storage/shortcut',
'PhabricatorRepositorySvnCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/svn',
'PhabricatorRepositorySvnCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/svn',
'PhabricatorRepositorySvnCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/svn',
'PhabricatorRepositorySymbol' => 'applications/repository/storage/symbol',
'PhabricatorRepositoryTestCase' => 'applications/repository/storage/repository/__tests__',
@ -1735,31 +1730,26 @@ phutil_register_library_map(array(
'PhabricatorRepositoryCommit' => 'PhabricatorRepositoryDAO',
'PhabricatorRepositoryCommitChangeParserWorker' => 'PhabricatorRepositoryCommitParserWorker',
'PhabricatorRepositoryCommitData' => 'PhabricatorRepositoryDAO',
'PhabricatorRepositoryCommitDiscoveryDaemon' => 'PhabricatorRepositoryDaemon',
'PhabricatorRepositoryCommitHeraldWorker' => 'PhabricatorRepositoryCommitParserWorker',
'PhabricatorRepositoryCommitMessageParserWorker' => 'PhabricatorRepositoryCommitParserWorker',
'PhabricatorRepositoryCommitOwnersWorker' => 'PhabricatorRepositoryCommitParserWorker',
'PhabricatorRepositoryCommitParserWorker' => 'PhabricatorWorker',
'PhabricatorRepositoryCommitTaskDaemon' => 'PhabricatorRepositoryDaemon',
'PhabricatorRepositoryCommitTaskDaemon' => 'PhabricatorDaemon',
'PhabricatorRepositoryController' => 'PhabricatorController',
'PhabricatorRepositoryCreateController' => 'PhabricatorRepositoryController',
'PhabricatorRepositoryDAO' => 'PhabricatorLiskDAO',
'PhabricatorRepositoryDaemon' => 'PhabricatorDaemon',
'PhabricatorRepositoryDefaultCommitMessageDetailParser' => 'PhabricatorRepositoryCommitMessageDetailParser',
'PhabricatorRepositoryDeleteController' => 'PhabricatorRepositoryController',
'PhabricatorRepositoryEditController' => 'PhabricatorRepositoryController',
'PhabricatorRepositoryGitCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
'PhabricatorRepositoryGitCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
'PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase' => 'PhabricatorTestCase',
'PhabricatorRepositoryGitCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
'PhabricatorRepositoryListController' => 'PhabricatorRepositoryController',
'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
'PhabricatorRepositoryMercurialCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
'PhabricatorRepositoryPullLocalDaemon' => 'PhabricatorDaemon',
'PhabricatorRepositoryPullLocalDaemonTestCase' => 'PhabricatorTestCase',
'PhabricatorRepositoryShortcut' => 'PhabricatorRepositoryDAO',
'PhabricatorRepositorySvnCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
'PhabricatorRepositorySvnCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
'PhabricatorRepositorySvnCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
'PhabricatorRepositorySymbol' => 'PhabricatorRepositoryDAO',
'PhabricatorRepositoryTestCase' => 'PhabricatorTestCase',

8
src/applications/diffusion/query/history/svn/DiffusionSvnHistoryQuery.php
View file

@ -1,7 +1,7 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -33,7 +33,11 @@ final class DiffusionSvnHistoryQuery extends DiffusionHistoryQuery {
PhabricatorRepository::TABLE_PATH,
array(md5('/'.trim($path, '/'))));
$paths = ipull($paths, 'id', 'path');
$path_id = $paths['/'.trim($path, '/')];
$path_id = idx($paths, '/'.trim($path, '/'));
if (!$path_id) {
return array();
}
$filter_query = '';
if ($this->needDirectChanges) {

37
src/applications/repository/controller/base/PhabricatorRepositoryController.php
View file

@ -1,7 +1,7 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -32,8 +32,43 @@ abstract class PhabricatorRepositoryController extends PhabricatorController {
$page->setGlyph("rX");
$page->appendChild($view);
$response = new AphrontWebpageResponse();
return $response->setContent($page->render());
}
private function isPullDaemonRunningOnThisMachine() {
// This is sort of hacky, but should probably work.
list($stdout) = execx('ps auxwww');
return preg_match('/PhabricatorRepositoryPullLocalDaemon/', $stdout);
}
protected function renderDaemonNotice() {
$daemon_running = $this->isPullDaemonRunningOnThisMachine();
if ($daemon_running) {
return null;
}
$documentation = phutil_render_tag(
'a',
array(
'href' => PhabricatorEnv::getDoclink(
'article/Diffusion_User_Guide.html'),
),
'Diffusion User Guide');
$view = new AphrontErrorView();
$view->setSeverity(AphrontErrorView::SEVERITY_WARNING);
$view->setTitle('Repository Daemon Not Running');
$view->appendChild(
"<p>The repository daemon is not running on this machine. Without this ".
"daemon, Phabricator will not be able to import or update repositories. ".
"For instructions on starting the daemon, see ".
"<strong>{$documentation}</strong>.</p>");
return $view;
}
}

4
src/applications/repository/controller/base/__init__.php
View file

@ -8,7 +8,11 @@
phutil_require_module('phabricator', 'aphront/response/webpage');
phutil_require_module('phabricator', 'applications/base/controller/base');
phutil_require_module('phabricator', 'infrastructure/env');
phutil_require_module('phabricator', 'view/form/error');
phutil_require_module('phutil', 'future/exec');
phutil_require_module('phutil', 'markup');
phutil_require_module('phutil', 'utils');

6
src/applications/repository/controller/edit/PhabricatorRepositoryEditController.php
View file

@ -63,6 +63,8 @@ final class PhabricatorRepositoryEditController
phutil_escape_html($name)));
}
$nav->appendChild($this->renderDaemonNotice());
$this->sideNav = $nav;
switch ($this->view) {
@ -345,9 +347,7 @@ final class PhabricatorRepositoryEditController
$error_view = new AphrontErrorView();
$error_view->setSeverity(AphrontErrorView::SEVERITY_NOTICE);
$error_view->setTitle('Changes Saved');
$error_view->appendChild(
'Tracking changes were saved. You may need to restart the daemon '.
'before changes will take effect.');
$error_view->appendChild('Tracking changes were saved.');
} else if (!$repository->isTracked()) {
$error_view = new AphrontErrorView();
$error_view->setSeverity(AphrontErrorView::SEVERITY_WARNING);

1
src/applications/repository/controller/list/PhabricatorRepositoryListController.php
View file

@ -157,6 +157,7 @@ final class PhabricatorRepositoryListController
return $this->buildStandardPageResponse(
array(
$this->renderDaemonNotice(),
$panel,
$project_panel,
),

38
src/applications/repository/daemon/base/PhabricatorRepositoryDaemon.php
View file

@ -1,38 +0,0 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
abstract class PhabricatorRepositoryDaemon extends PhabricatorDaemon {
protected function loadRepository() {
$argv = $this->getArgv();
if (count($argv) !== 1) {
throw new Exception("No repository PHID provided!");
}
$repository = id(new PhabricatorRepository())->loadOneWhere(
'phid = %s',
$argv[0]);
if (!$repository) {
throw new Exception("No such repository exists!");
}
return $repository;
}
}

15
src/applications/repository/daemon/base/__init__.php
View file

@ -1,15 +0,0 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_module('phabricator', 'applications/repository/storage/repository');
phutil_require_module('phabricator', 'infrastructure/daemon/base');
phutil_require_module('phutil', 'utils');
phutil_require_source('PhabricatorRepositoryDaemon.php');

113
src/applications/repository/daemon/commitdiscovery/base/PhabricatorRepositoryCommitDiscoveryDaemon.php
View file

@ -1,113 +0,0 @@
<?php
/*
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
abstract class PhabricatorRepositoryCommitDiscoveryDaemon
extends PhabricatorRepositoryDaemon {
private $repository;
private $commitCache = array();
final protected function getRepository() {
return $this->repository;
}
final public function run() {
while (true) {
// Reload the repository every time to pick up changes from the web
// console.
$this->repository = $this->loadRepository();
$this->discoverCommits();
$sleep = max(2, $this->getRepository()->getDetail('pull-frequency'));
$this->sleep($sleep);
}
}
final public function runOnce() {
$this->repository = $this->loadRepository();
$this->discoverCommits();
}
protected function isKnownCommit($target) {
if (isset($this->commitCache[$target])) {
return true;
}
$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
'repositoryID = %s AND commitIdentifier = %s',
$this->getRepository()->getID(),
$target);
if (!$commit) {
return false;
}
$this->commitCache[$target] = true;
while (count($this->commitCache) > 64) {
array_shift($this->commitCache);
}
return true;
}
protected function recordCommit($commit_identifier, $epoch) {
$repository = $this->getRepository();
$commit = new PhabricatorRepositoryCommit();
$commit->setRepositoryID($repository->getID());
$commit->setCommitIdentifier($commit_identifier);
$commit->setEpoch($epoch);
try {
$commit->save();
$event = new PhabricatorTimelineEvent(
'cmit',
array(
'id' => $commit->getID(),
));
$event->recordEvent();
queryfx(
$repository->establishConnection('w'),
'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
VALUES (%d, 1, %d, %d)
ON DUPLICATE KEY UPDATE
size = size + 1,
lastCommitID =
IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID),
epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)',
PhabricatorRepository::TABLE_SUMMARY,
$repository->getID(),
$commit->getID(),
$epoch);
$this->commitCache[$commit_identifier] = true;
} catch (AphrontQueryDuplicateKeyException $ex) {
// Ignore. This can happen because we discover the same new commit
// more than once when looking at history, or because of races or
// data inconsistency or cosmic radiation; in any case, we're still
// in a good state if we ignore the failure.
$this->commitCache[$commit_identifier] = true;
}
$this->stillWorking();
}
abstract protected function discoverCommits();
}

18
src/applications/repository/daemon/commitdiscovery/base/__init__.php
View file

@ -1,18 +0,0 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_module('phabricator', 'applications/repository/daemon/base');
phutil_require_module('phabricator', 'applications/repository/storage/commit');
phutil_require_module('phabricator', 'applications/repository/storage/repository');
phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/event');
phutil_require_module('phabricator', 'storage/queryfx');
phutil_require_module('phutil', 'utils');
phutil_require_source('PhabricatorRepositoryCommitDiscoveryDaemon.php');

162
src/applications/repository/daemon/commitdiscovery/git/PhabricatorRepositoryGitCommitDiscoveryDaemon.php
View file

@ -1,162 +0,0 @@
<?php
/*
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
final class PhabricatorRepositoryGitCommitDiscoveryDaemon
extends PhabricatorRepositoryCommitDiscoveryDaemon {
protected function discoverCommits() {
// NOTE: PhabricatorRepositoryGitFetchDaemon does the actual pulls, this
// just parses HEAD.
$repository = $this->getRepository();
$vcs = $repository->getVersionControlSystem();
if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_GIT) {
throw new Exception("Repository is not a git repository.");
}
list($remotes) = $repository->execxLocalCommand(
'remote show -n origin');
$matches = null;
if (!preg_match('/^\s*Fetch URL:\s*(.*?)\s*$/m', $remotes, $matches)) {
throw new Exception(
"Expected 'Fetch URL' in 'git remote show -n origin'.");
}
self::verifySameGitOrigin(
$matches[1],
$repository->getRemoteURI(),
$repository->getLocalPath());
list($stdout) = $repository->execxLocalCommand(
'branch -r --verbose --no-abbrev');
$branches = DiffusionGitBranchQuery::parseGitRemoteBranchOutput(
$stdout,
$only_this_remote = DiffusionBranchInformation::DEFAULT_GIT_REMOTE);
$got_something = false;
$tracked_something = false;
foreach ($branches as $name => $commit) {
if (!$repository->shouldTrackBranch($name)) {
continue;
}
$tracked_something = true;
if ($this->isKnownCommit($commit)) {
continue;
} else {
$this->discoverCommit($commit);
$got_something = true;
}
}
if (!$tracked_something) {
$repo_name = $repository->getName();
$repo_callsign = $repository->getCallsign();
throw new Exception(
"Repository r{$repo_callsign} '{$repo_name}' has no tracked branches! ".
"Verify that your branch filtering settings are correct.");
}
return $got_something;
}
private function discoverCommit($commit) {
$discover = array();
$insert = array();
$repository = $this->getRepository();
$discover[] = $commit;
$insert[] = $commit;
$seen_parent = array();
while (true) {
$target = array_pop($discover);
list($parents) = $repository->execxLocalCommand(
'log -n1 --pretty="%%P" %s',
$target);
$parents = array_filter(explode(' ', trim($parents)));
foreach ($parents as $parent) {
if (isset($seen_parent[$parent])) {
// We end up in a loop here somehow when we parse Arcanist if we
// don't do this. TODO: Figure out why and draw a pretty diagram
// since it's not evident how parsing a DAG with this causes the
// loop to stop terminating.
continue;
}
$seen_parent[$parent] = true;
if (!$this->isKnownCommit($parent)) {
$discover[] = $parent;
$insert[] = $parent;
}
}
if (empty($discover)) {
break;
}
$this->stillWorking();
}
while (true) {
$target = array_pop($insert);
list($epoch) = $repository->execxLocalCommand(
'log -n1 --pretty="%%ct" %s',
$target);
$epoch = trim($epoch);
$this->recordCommit($target, $epoch);
if (empty($insert)) {
break;
}
}
}
public static function verifySameGitOrigin($remote, $expect, $where) {
$remote_uri = PhabricatorRepository::newPhutilURIFromGitURI($remote);
$expect_uri = PhabricatorRepository::newPhutilURIFromGitURI($expect);
$remote_path = $remote_uri->getPath();
$expect_path = $expect_uri->getPath();
$remote_match = self::normalizeGitPath($remote_path);
$expect_match = self::normalizeGitPath($expect_path);
if ($remote_match != $expect_match) {
throw new Exception(
"Working copy at '{$where}' has a mismatched origin URL. It has ".
"origin URL '{$remote}' (with remote path '{$remote_path}'), but the ".
"configured URL '{$expect}' (with remote path '{$expect_path}') is ".
"expected. Refusing to proceed because this may indicate that the ".
"working copy is actually some other repository.");
}
}
private static function normalizeGitPath($path) {
// Strip away trailing "/" and ".git", so similar paths correctly match.
$path = rtrim($path, '/');
$path = preg_replace('/\.git$/', '', $path);
return $path;
}
}

16
src/applications/repository/daemon/commitdiscovery/git/__init__.php
View file

@ -1,16 +0,0 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_module('phabricator', 'applications/diffusion/data/branch');
phutil_require_module('phabricator', 'applications/diffusion/query/branch/git');
phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
phutil_require_module('phabricator', 'applications/repository/daemon/commitdiscovery/base');
phutil_require_module('phabricator', 'applications/repository/storage/repository');
phutil_require_source('PhabricatorRepositoryGitCommitDiscoveryDaemon.php');

137
src/applications/repository/daemon/commitdiscovery/mercurial/PhabricatorRepositoryMercurialCommitDiscoveryDaemon.php
View file

@ -1,137 +0,0 @@
<?php
/*
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
final class PhabricatorRepositoryMercurialCommitDiscoveryDaemon
extends PhabricatorRepositoryCommitDiscoveryDaemon {
protected function discoverCommits() {
$repository = $this->getRepository();
$vcs = $repository->getVersionControlSystem();
if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL) {
throw new Exception("Repository is not a Mercurial repository.");
}
$repository_phid = $repository->getPHID();
list($stdout) = $repository->execxLocalCommand('branches');
$branches = ArcanistMercurialParser::parseMercurialBranches($stdout);
$got_something = false;
foreach ($branches as $name => $branch) {
$commit = $branch['rev'];
$commit = $this->getFullHash($commit);
if ($this->isKnownCommit($commit)) {
continue;
} else {
$this->discoverCommit($commit);
$got_something = true;
}
}
return $got_something;
}
private function getFullHash($commit) {
// NOTE: Mercurial shortens hashes to 12 characters by default. This
// implies collisions with as few as a few million commits. The
// documentation sensibly advises "Do not use short-form IDs for
// long-lived representations". It then continues "You can use the
// --debug option to display the full changeset ID". What?! Yes, this
// is in fact the only way to turn on full hashes, and the hg source
// code is littered with "hexfn = ui.debugflag and hex or short" and
// similar. There is no more-selective flag or config option.
//
// Unfortunately, "hg --debug" turns on tons of other extra output,
// including full commit messages in "hg log" and "hg parents" (which
// ignore --style); this renders them unparseable. So we have to use
// "hg id" to convert short hashes into full hashes. See:
//
// <http://mercurial.selenic.com/wiki/ChangeSetID>
//
// Of course, this means that if there are collisions we will break here
// (the short commit identifier won't be unambiguous) but maybe Mercurial
// will have a --full-hashes flag or something by then and we can fix it
// properly. Until we run into that, this allows us to store data in the
// right format so when we eventually encounter this we won't have to
// reparse every Mercurial repository.
$repository = $this->getRepository();
list($stdout) = $repository->execxLocalCommand(
'id --debug -i --rev %s',
$commit);
return trim($stdout);
}
private function discoverCommit($commit) {
$discover = array();
$insert = array();
$repository = $this->getRepository();
$discover[] = $commit;
$insert[] = $commit;
$seen_parent = array();
// For all the new commits at the branch heads, walk backward until we find
// only commits we've aleady seen.
while (true) {
$target = array_pop($discover);
list($stdout) = $repository->execxLocalCommand(
'parents --style default --rev %s',
$target);
$parents = ArcanistMercurialParser::parseMercurialLog($stdout);
if ($parents) {
foreach ($parents as $parent) {
$parent_commit = $parent['rev'];
$parent_commit = $this->getFullHash($parent_commit);
if (isset($seen_parent[$parent_commit])) {
continue;
}
$seen_parent[$parent_commit] = true;
if (!$this->isKnownCommit($parent_commit)) {
$discover[] = $parent_commit;
$insert[] = $parent_commit;
}
}
}
if (empty($discover)) {
break;
}
$this->stillWorking();
}
while (true) {
$target = array_pop($insert);
list($stdout) = $repository->execxLocalCommand(
'log --rev %s --template %s',
$target,
'{date|rfc822date}');
$epoch = strtotime($stdout);
$this->recordCommit($target, $epoch);
if (empty($insert)) {
break;
}
}
}
}

15
src/applications/repository/daemon/commitdiscovery/mercurial/__init__.php
View file

@ -1,15 +0,0 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_module('arcanist', 'repository/parser/mercurial');
phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
phutil_require_module('phabricator', 'applications/repository/daemon/commitdiscovery/base');
phutil_require_source('PhabricatorRepositoryMercurialCommitDiscoveryDaemon.php');

123
src/applications/repository/daemon/commitdiscovery/svn/PhabricatorRepositorySvnCommitDiscoveryDaemon.php
View file

@ -1,123 +0,0 @@
<?php
/*
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
final class PhabricatorRepositorySvnCommitDiscoveryDaemon
extends PhabricatorRepositoryCommitDiscoveryDaemon {
protected function discoverCommits() {
$repository = $this->getRepository();
$vcs = $repository->getVersionControlSystem();
if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_SVN) {
throw new Exception("Repository is not a svn repository.");
}
$uri = $this->getBaseSVNLogURI();
list($xml) = $repository->execxRemoteCommand(
'log --xml --quiet --limit 1 %s@HEAD',
$uri);
$results = $this->parseSVNLogXML($xml);
$commit = head_key($results);
$epoch = head($results);
if ($this->isKnownCommit($commit)) {
return false;
}
$this->discoverCommit($commit, $epoch);
return true;
}
private function discoverCommit($commit, $epoch) {
$uri = $this->getBaseSVNLogURI();
$repository = $this->getRepository();
$discover = array(
$commit => $epoch,
);
$upper_bound = $commit;
$limit = 1;
while ($upper_bound > 1 && !$this->isKnownCommit($upper_bound)) {
// Find all the unknown commits on this path. Note that we permit
// importing an SVN subdirectory rather than the entire repository, so
// commits may be nonsequential.
list($err, $xml, $stderr) = $repository->execRemoteCommand(
' log --xml --quiet --limit %d %s@%d',
$limit,
$uri,
$upper_bound - 1);
if ($err) {
if (preg_match('/(path|File) not found/', $stderr)) {
// We've gone all the way back through history and this path was not
// affected by earlier commits.
break;
} else {
throw new Exception("svn log error #{$err}: {$stderr}");
}
}
$discover += $this->parseSVNLogXML($xml);
$upper_bound = min(array_keys($discover));
// Discover 2, 4, 8, ... 256 logs at a time. This allows us to initially
// import large repositories fairly quickly, while pulling only as much
// data as we need in the common case (when we've already imported the
// repository and are just grabbing one commit at a time).
$limit = min($limit * 2, 256);
}
// NOTE: We do writes only after discovering all the commits so that we're
// never left in a state where we've missed commits -- if the discovery
// script terminates it can always resume and restore the import to a good
// state. This is also why we sort the discovered commits so we can do
// writes forward from the smallest one.
ksort($discover);
foreach ($discover as $commit => $epoch) {
$this->recordCommit($commit, $epoch);
}
}
private function parseSVNLogXML($xml) {
$xml = phutil_utf8ize($xml);
$result = array();
$log = new SimpleXMLElement($xml);
foreach ($log->logentry as $entry) {
$commit = (int)$entry['revision'];
$epoch = (int)strtotime((string)$entry->date[0]);
$result[$commit] = $epoch;
}
return $result;
}
private function getBaseSVNLogURI() {
$repository = $this->getRepository();
$uri = $repository->getDetail('remote-uri');
$subpath = $repository->getDetail('svn-subpath');
return $uri.$subpath;
}
}

15
src/applications/repository/daemon/commitdiscovery/svn/__init__.php
View file

@ -1,15 +0,0 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
phutil_require_module('phabricator', 'applications/repository/daemon/commitdiscovery/base');
phutil_require_module('phutil', 'utils');
phutil_require_source('PhabricatorRepositorySvnCommitDiscoveryDaemon.php');

2
src/applications/repository/daemon/committask/PhabricatorRepositoryCommitTaskDaemon.php
View file

@ -17,7 +17,7 @@
*/
final class PhabricatorRepositoryCommitTaskDaemon
extends PhabricatorRepositoryDaemon {
extends PhabricatorDaemon {
final public function run() {
do {

2
src/applications/repository/daemon/committask/__init__.php
View file

@ -7,9 +7,9 @@
phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
phutil_require_module('phabricator', 'applications/repository/daemon/base');
phutil_require_module('phabricator', 'applications/repository/storage/commit');
phutil_require_module('phabricator', 'applications/repository/storage/repository');
phutil_require_module('phabricator', 'infrastructure/daemon/base');
phutil_require_module('phabricator', 'infrastructure/daemon/timeline/cursor/iterator');
phutil_require_module('phabricator', 'infrastructure/daemon/workers/storage/task');

520
src/applications/repository/daemon/pulllocal/PhabricatorRepositoryPullLocalDaemon.php
View file

@ -24,7 +24,12 @@
* responsible for only some repositories, you can launch it with a list of
* PHIDs or callsigns:
*
* ./phd launch repositorypulllocal X Q Z
* ./phd launch repositorypulllocal -- X Q Z
*
* You can also launch a daemon which is responsible for all //but// one or
* more repositories:
*
* ./phd launch repositorypulllocal -- --not A --not B
*
* If you have a very large number of repositories and some aren't being pulled
* as frequently as you'd like, you can either change the pull frequency of
@ -40,6 +45,8 @@
final class PhabricatorRepositoryPullLocalDaemon
extends PhabricatorDaemon {
private static $commitCache = array();
/* -( Pulling Repositories )----------------------------------------------- */
@ -48,14 +55,45 @@ final class PhabricatorRepositoryPullLocalDaemon
* @task pull
*/
public function run() {
$argv = $this->getArgv();
array_unshift($argv, __CLASS__);
$args = new PhutilArgumentParser($argv);
$args->parse(
array(
array(
'name' => 'no-discovery',
'help' => 'Pull only, without discovering commits.',
),
array(
'name' => 'not',
'param' => 'repository',
'repeat' => true,
'help' => 'Do not pull __repository__.',
),
array(
'name' => 'repositories',
'wildcard' => true,
'help' => 'Pull specific __repositories__ instead of all.',
),
));
$no_discovery = $args->getArg('no-discovery');
$repo_names = $args->getArg('repositories', array());
$exclude_names = $args->getArg('not', array());
// Each repository has an individual pull frequency; after we pull it,
// wait that long to pull it again. When we start up, try to pull everything
// serially.
$retry_after = array();
$min_sleep = 15;
while (true) {
$repositories = $this->loadRepositories();
$repositories = $this->loadRepositories($repo_names);
if ($exclude_names) {
$exclude = $this->loadRepositories($exclude_names);
$repositories = array_diff_key($repositories, $exclude);
}
// Shuffle the repositories, then re-key the array since shuffle()
// discards keys. This is mostly for startup, we'll use soft priorities
@ -79,22 +117,41 @@ final class PhabricatorRepositoryPullLocalDaemon
foreach ($repositories as $id => $repository) {
$after = idx($retry_after, $id, 0);
if ($after >= time()) {
if ($after > time()) {
continue;
}
$tracked = $repository->isTracked();
if (!$tracked) {
continue;
}
try {
self::pullRepository($repository);
$sleep_for = $repository->getDetail('pull-frequency', 15);
$retry_after[$id] = time() + $sleep_for;
} catch (Exception $ex) {
$retry_after[$id] = time() + 15;
phlog($ex);
}
if (!$no_discovery) {
// TODO: It would be nice to discover only if we pulled something,
// but this isn't totally trivial.
self::discoverRepository($repository);
}
$sleep_until = max(min($retry_after), time() + 15);
sleep($sleep_until - time());
$sleep_for = $repository->getDetail('pull-frequency', $min_sleep);
$retry_after[$id] = time() + $sleep_for;
} catch (Exception $ex) {
$retry_after[$id] = time() + $min_sleep;
phlog($ex);
}
$this->stillWorking();
}
if ($retry_after) {
$sleep_until = max(min($retry_after), time() + $min_sleep);
} else {
$sleep_until = time() + $min_sleep;
}
$this->sleep($sleep_until - time());
}
}
@ -102,12 +159,11 @@ final class PhabricatorRepositoryPullLocalDaemon
/**
* @task pull
*/
protected function loadRepositories() {
$argv = $this->getArgv();
if (!count($argv)) {
protected function loadRepositories(array $names) {
if (!count($names)) {
return id(new PhabricatorRepository())->loadAll();
} else {
return PhabricatorRepository::loadAllByPHIDOrCallsign($argv);
return PhabricatorRepository::loadAllByPHIDOrCallsign($names);
}
}
@ -116,11 +172,6 @@ final class PhabricatorRepositoryPullLocalDaemon
* @task pull
*/
public static function pullRepository(PhabricatorRepository $repository) {
$tracked = $repository->isTracked();
if (!$tracked) {
return;
}
$vcs = $repository->getVersionControlSystem();
$is_svn = ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_SVN);
@ -153,19 +204,126 @@ final class PhabricatorRepositoryPullLocalDaemon
}
if ($is_git) {
self::executeGitCreate($repository, $local_path);
return self::executeGitCreate($repository, $local_path);
} else if ($is_hg) {
self::executeHgCreate($repository, $local_path);
return self::executeHgCreate($repository, $local_path);
}
} else {
if ($is_git) {
self::executeGitUpdate($repository, $local_path);
return self::executeGitUpdate($repository, $local_path);
} else if ($is_hg) {
self::executeHgUpdate($repository, $local_path);
return self::executeHgUpdate($repository, $local_path);
}
}
}
public static function discoverRepository(PhabricatorRepository $repository) {
$vcs = $repository->getVersionControlSystem();
switch ($vcs) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
return self::executeGitDiscover($repository);
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
return self::executeSvnDiscover($repository);
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
return self::executeHgDiscover($repository);
default:
throw new Exception("Unknown VCS '{$vcs}'!");
}
}
private static function isKnownCommit(
PhabricatorRepository $repository,
$target) {
if (self::getCache($repository, $target)) {
return true;
}
$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
'repositoryID = %s AND commitIdentifier = %s',
$repository->getID(),
$target);
if (!$commit) {
return false;
}
self::setCache($repository, $target);
while (count(self::$commitCache) > 2048) {
array_shift(self::$commitCache);
}
return true;
}
private static function recordCommit(
PhabricatorRepository $repository,
$commit_identifier,
$epoch) {
$commit = new PhabricatorRepositoryCommit();
$commit->setRepositoryID($repository->getID());
$commit->setCommitIdentifier($commit_identifier);
$commit->setEpoch($epoch);
try {
$commit->save();
$event = new PhabricatorTimelineEvent(
'cmit',
array(
'id' => $commit->getID(),
));
$event->recordEvent();
queryfx(
$repository->establishConnection('w'),
'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
VALUES (%d, 1, %d, %d)
ON DUPLICATE KEY UPDATE
size = size + 1,
lastCommitID =
IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID),
epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)',
PhabricatorRepository::TABLE_SUMMARY,
$repository->getID(),
$commit->getID(),
$epoch);
self::setCache($repository, $commit_identifier);
} catch (AphrontQueryDuplicateKeyException $ex) {
// Ignore. This can happen because we discover the same new commit
// more than once when looking at history, or because of races or
// data inconsistency or cosmic radiation; in any case, we're still
// in a good state if we ignore the failure.
self::setCache($repository, $commit_identifier);
}
}
private static function setCache(
PhabricatorRepository $repository,
$commit_identifier) {
$key = self::getCacheKey($repository, $commit_identifier);
self::$commitCache[$key] = true;
}
private static function getCache(
PhabricatorRepository $repository,
$commit_identifier) {
$key = self::getCacheKey($repository, $commit_identifier);
return idx(self::$commitCache, $key, false);
}
private static function getCacheKey(
PhabricatorRepository $repository,
$commit_identifier) {
return $repository->getID().':'.$commit_identifier;
}
/* -( Git Implementation )------------------------------------------------- */
@ -249,6 +407,147 @@ final class PhabricatorRepositoryPullLocalDaemon
}
/**
* @task git
*/
private static function executeGitDiscover(
PhabricatorRepository $repository) {
list($remotes) = $repository->execxLocalCommand(
'remote show -n origin');
$matches = null;
if (!preg_match('/^\s*Fetch URL:\s*(.*?)\s*$/m', $remotes, $matches)) {
throw new Exception(
"Expected 'Fetch URL' in 'git remote show -n origin'.");
}
self::executeGitverifySameOrigin(
$matches[1],
$repository->getRemoteURI(),
$repository->getLocalPath());
list($stdout) = $repository->execxLocalCommand(
'branch -r --verbose --no-abbrev');
$branches = DiffusionGitBranchQuery::parseGitRemoteBranchOutput(
$stdout,
$only_this_remote = DiffusionBranchInformation::DEFAULT_GIT_REMOTE);
$tracked_something = false;
foreach ($branches as $name => $commit) {
if (!$repository->shouldTrackBranch($name)) {
continue;
}
$tracked_something = true;
if (self::isKnownCommit($repository, $commit)) {
continue;
} else {
self::executeGitDiscoverCommit($repository, $commit);
}
}
if (!$tracked_something) {
$repo_name = $repository->getName();
$repo_callsign = $repository->getCallsign();
throw new Exception(
"Repository r{$repo_callsign} '{$repo_name}' has no tracked branches! ".
"Verify that your branch filtering settings are correct.");
}
}
/**
* @task git
*/
private static function executeGitDiscoverCommit(
PhabricatorRepository $repository,
$commit) {
$discover = array($commit);
$insert = array($commit);
$seen_parent = array();
while (true) {
$target = array_pop($discover);
list($parents) = $repository->execxLocalCommand(
'log -n1 --pretty="%%P" %s',
$target);
$parents = array_filter(explode(' ', trim($parents)));
foreach ($parents as $parent) {
if (isset($seen_parent[$parent])) {
// We end up in a loop here somehow when we parse Arcanist if we
// don't do this. TODO: Figure out why and draw a pretty diagram
// since it's not evident how parsing a DAG with this causes the
// loop to stop terminating.
continue;
}
$seen_parent[$parent] = true;
if (!self::isKnownCommit($repository, $parent)) {
$discover[] = $parent;
$insert[] = $parent;
}
}
if (empty($discover)) {
break;
}
}
while (true) {
$target = array_pop($insert);
list($epoch) = $repository->execxLocalCommand(
'log -n1 --pretty="%%ct" %s',
$target);
$epoch = trim($epoch);
self::recordCommit($repository, $target, $epoch);
if (empty($insert)) {
break;
}
}
}
/**
* @task git
*/
public static function executeGitVerifySameOrigin($remote, $expect, $where) {
$remote_uri = PhabricatorRepository::newPhutilURIFromGitURI($remote);
$expect_uri = PhabricatorRepository::newPhutilURIFromGitURI($expect);
$remote_path = $remote_uri->getPath();
$expect_path = $expect_uri->getPath();
$remote_match = self::executeGitNormalizePath($remote_path);
$expect_match = self::executeGitNormalizePath($expect_path);
if ($remote_match != $expect_match) {
throw new Exception(
"Working copy at '{$where}' has a mismatched origin URL. It has ".
"origin URL '{$remote}' (with remote path '{$remote_path}'), but the ".
"configured URL '{$expect}' (with remote path '{$expect_path}') is ".
"expected. Refusing to proceed because this may indicate that the ".
"working copy is actually some other repository.");
}
}
/**
* @task git
*/
private static function executeGitNormalizePath($path) {
// Strip away trailing "/" and ".git", so similar paths correctly match.
$path = rtrim($path, '/');
$path = preg_replace('/\.git$/', '', $path);
return $path;
}
/* -( Mercurial Implementation )------------------------------------------- */
@ -306,4 +605,177 @@ final class PhabricatorRepositoryPullLocalDaemon
}
}
private static function executeHgDiscover(PhabricatorRepository $repository) {
// NOTE: "--debug" gives us 40-character hashes.
list($stdout) = $repository->execxLocalCommand('--debug branches');
$branches = ArcanistMercurialParser::parseMercurialBranches($stdout);
$got_something = false;
foreach ($branches as $name => $branch) {
$commit = $branch['rev'];
if (self::isKnownCommit($repository, $commit)) {
continue;
} else {
self::executeHgDiscoverCommit($repository, $commit);
$got_something = true;
}
}
return $got_something;
}
private static function executeHgDiscoverCommit(
PhabricatorRepository $repository,
$commit) {
$discover = array($commit);
$insert = array($commit);
$seen_parent = array();
// For all the new commits at the branch heads, walk backward until we find
// only commits we've aleady seen.
while (true) {
$target = array_pop($discover);
list($stdout) = $repository->execxLocalCommand(
'parents --rev %s --template %s',
$target,
'{node}\n');
$parents = array_filter(explode("\n", trim($stdout)));
foreach ($parents as $parent) {
if (isset($seen_parent[$parent])) {
continue;
}
$seen_parent[$parent] = true;
if (!self::isKnownCommit($repository, $parent)) {
$discover[] = $parent;
$insert[] = $parent;
}
}
if (empty($discover)) {
break;
}
}
while (true) {
$target = array_pop($insert);
list($stdout) = $repository->execxLocalCommand(
'log --rev %s --template %s',
$target,
'{date|rfc822date}');
$epoch = strtotime($stdout);
self::recordCommit($repository, $target, $epoch);
if (empty($insert)) {
break;
}
}
}
/* -( Subversion Implementation )------------------------------------------ */
private static function executeSvnDiscover(
PhabricatorRepository $repository) {
$uri = self::executeSvnGetBaseSVNLogURI($repository);
list($xml) = $repository->execxRemoteCommand(
'log --xml --quiet --limit 1 %s@HEAD',
$uri);
$results = self::executeSvnParseLogXML($xml);
$commit = head_key($results);
$epoch = head($results);
if (self::isKnownCommit($repository, $commit)) {
return false;
}
self::executeSvnDiscoverCommit($repository, $commit, $epoch);
return true;
}
private static function executeSvnDiscoverCommit(
PhabricatorRepository $repository,
$commit,
$epoch) {
$uri = self::executeSvnGetBaseSVNLogURI($repository);
$discover = array(
$commit => $epoch,
);
$upper_bound = $commit;
$limit = 1;
while ($upper_bound > 1 &&
!self::isKnownCommit($repository, $upper_bound)) {
// Find all the unknown commits on this path. Note that we permit
// importing an SVN subdirectory rather than the entire repository, so
// commits may be nonsequential.
list($err, $xml, $stderr) = $repository->execRemoteCommand(
' log --xml --quiet --limit %d %s@%d',
$limit,
$uri,
$upper_bound - 1);
if ($err) {
if (preg_match('/(path|File) not found/', $stderr)) {
// We've gone all the way back through history and this path was not
// affected by earlier commits.
break;
} else {
throw new Exception("svn log error #{$err}: {$stderr}");
}
}
$discover += self::executeSvnParseLogXML($xml);
$upper_bound = min(array_keys($discover));
// Discover 2, 4, 8, ... 256 logs at a time. This allows us to initially
// import large repositories fairly quickly, while pulling only as much
// data as we need in the common case (when we've already imported the
// repository and are just grabbing one commit at a time).
$limit = min($limit * 2, 256);
}
// NOTE: We do writes only after discovering all the commits so that we're
// never left in a state where we've missed commits -- if the discovery
// script terminates it can always resume and restore the import to a good
// state. This is also why we sort the discovered commits so we can do
// writes forward from the smallest one.
ksort($discover);
foreach ($discover as $commit => $epoch) {
self::recordCommit($repository, $commit, $epoch);
}
}
private static function executeSvnParseLogXML($xml) {
$xml = phutil_utf8ize($xml);
$result = array();
$log = new SimpleXMLElement($xml);
foreach ($log->logentry as $entry) {
$commit = (int)$entry['revision'];
$epoch = (int)strtotime((string)$entry->date[0]);
$result[$commit] = $epoch;
}
return $result;
}
private static function executeSvnGetBaseSVNLogURI(
PhabricatorRepository $repository) {
$uri = $repository->getDetail('remote-uri');
$subpath = $repository->getDetail('svn-subpath');
return $uri.$subpath;
}
}

8
src/applications/repository/daemon/pulllocal/__init__.php
View file

@ -6,12 +6,20 @@
phutil_require_module('arcanist', 'repository/parser/mercurial');
phutil_require_module('phabricator', 'applications/diffusion/data/branch');
phutil_require_module('phabricator', 'applications/diffusion/query/branch/git');
phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
phutil_require_module('phabricator', 'applications/repository/storage/commit');
phutil_require_module('phabricator', 'applications/repository/storage/repository');
phutil_require_module('phabricator', 'infrastructure/daemon/base');
phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/event');
phutil_require_module('phabricator', 'storage/queryfx');
phutil_require_module('phutil', 'error');
phutil_require_module('phutil', 'filesystem');
phutil_require_module('phutil', 'parser/argument/parser');
phutil_require_module('phutil', 'utils');

6
src/applications/repository/daemon/commitdiscovery/git/__tests__/PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase.php → src/applications/repository/daemon/pulllocal/__tests__/PhabricatorRepositoryPullLocalDaemonTestCase.php
View file

@ -16,10 +16,10 @@
* limitations under the License.
*/
final class PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase
final class PhabricatorRepositoryPullLocalDaemonTestCase
extends PhabricatorTestCase {
public function testVerifySameGitOrigin() {
public function testExecuteGitVerifySameOrigin() {
$cases = array(
array(
'ssh://user@domain.com/path.git',
@ -94,7 +94,7 @@ final class PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase
$ex = null;
try {
PhabricatorRepositoryGitCommitDiscoveryDaemon::verifySameGitOrigin(
PhabricatorRepositoryPullLocalDaemon::executeGitverifySameOrigin(
$remote,
$config,
'(a test case)');

4
src/applications/repository/daemon/commitdiscovery/git/__tests__/__init__.php → src/applications/repository/daemon/pulllocal/__tests__/__init__.php
View file

@ -6,8 +6,8 @@
phutil_require_module('phabricator', 'applications/repository/daemon/commitdiscovery/git');
phutil_require_module('phabricator', 'applications/repository/daemon/pulllocal');
phutil_require_module('phabricator', 'infrastructure/testing/testcase');
phutil_require_source('PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase.php');
phutil_require_source('PhabricatorRepositoryPullLocalDaemonTestCase.php');