mirror of
https://we.phorge.it/source/phorge.git
synced 2024-11-25 16:22:43 +01:00
Use one daemon to discover commits in all repositories, not one per repository
Summary: See D2418. This merges the commit discovery daemon into the same single daemon, and applies all the same rules to it. There are relatively few implementation changes, but a few things did change: - I simplified/improved Mercurial importing, by finding full branch tip hashes with "--debug branches" and using "parents --template {node}" so we don't need to do separate "--debug id" calls. - Added a new "--not" flag to exclude repositories, since I switched to real arg parsing anyway. - I removed a web UI notification that you need to restart the daemons, this is no longer true. - I added a web UI notification that no pull daemon is running on the machine. NOTE: @makinde, this doesn't change anything from your perspective, but it something breaks this is the likely cause. This implicitly resolves T792, because discovery no longer runs before pulling. Test Plan: - Swapped databases to a fresh install. - Ran "pulllocal" in debug mode. Verified it correctly does nothing (fixed a minor issue with min() on empty array). - Added an SVN repository. Verified it cloned and discovered correctly. - Added a Mercurial repository. Verified it cloned and discovered correctly. - Added a Git repository. Verified it cloned and discovered correctly. - Ran with arguments to verify behaviors: "--not MTEST --not STEST", "P --no-discovery", "P". Reviewers: btrahan, csilvers, Makinde Reviewed By: btrahan CC: aran Maniphest Tasks: T792 Differential Revision: https://secure.phabricator.com/D2430
This commit is contained in:
parent
679f778235
commit
d2b01aead0
24 changed files with 637 additions and 748 deletions
|
@ -52,13 +52,20 @@ switch (isset($argv[1]) ? $argv[1] : 'help') {
|
||||||
$need_launch = phd_load_tracked_repositories();
|
$need_launch = phd_load_tracked_repositories();
|
||||||
if (!$need_launch) {
|
if (!$need_launch) {
|
||||||
echo "There are no repositories with tracking enabled.\n";
|
echo "There are no repositories with tracking enabled.\n";
|
||||||
exit(0);
|
exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
will_launch($control);
|
will_launch($control);
|
||||||
|
|
||||||
|
echo "Launching PullLocal daemon in readonly mode...\n";
|
||||||
|
|
||||||
$control->launchDaemon(
|
$control->launchDaemon(
|
||||||
'PhabricatorRepositoryPullLocalDaemon',
|
'PhabricatorRepositoryPullLocalDaemon',
|
||||||
array());
|
array(
|
||||||
|
'--no-discovery',
|
||||||
|
));
|
||||||
|
|
||||||
|
echo "Done.\n";
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case 'repository-launch-master':
|
case 'repository-launch-master':
|
||||||
|
@ -66,55 +73,24 @@ switch (isset($argv[1]) ? $argv[1] : 'help') {
|
||||||
if (!$need_launch) {
|
if (!$need_launch) {
|
||||||
echo "There are no repositories with tracking enabled.\n";
|
echo "There are no repositories with tracking enabled.\n";
|
||||||
exit(1);
|
exit(1);
|
||||||
} else {
|
|
||||||
will_launch($control);
|
|
||||||
|
|
||||||
$control->launchDaemon(
|
|
||||||
'PhabricatorRepositoryPullLocalDaemon',
|
|
||||||
array());
|
|
||||||
|
|
||||||
foreach ($need_launch as $repository) {
|
|
||||||
$name = $repository->getName();
|
|
||||||
$callsign = $repository->getCallsign();
|
|
||||||
$desc = "'{$name}' ({$callsign})";
|
|
||||||
$phid = $repository->getPHID();
|
|
||||||
|
|
||||||
switch ($repository->getVersionControlSystem()) {
|
|
||||||
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
|
|
||||||
echo "Launching discovery daemon on the {$desc} repository...\n";
|
|
||||||
$control->launchDaemon(
|
|
||||||
'PhabricatorRepositoryGitCommitDiscoveryDaemon',
|
|
||||||
array(
|
|
||||||
$phid,
|
|
||||||
));
|
|
||||||
break;
|
|
||||||
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
|
|
||||||
echo "Launching discovery daemon on the {$desc} repository...\n";
|
|
||||||
$control->launchDaemon(
|
|
||||||
'PhabricatorRepositorySvnCommitDiscoveryDaemon',
|
|
||||||
array(
|
|
||||||
$phid,
|
|
||||||
));
|
|
||||||
break;
|
|
||||||
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
|
|
||||||
echo "Launching discovery daemon on the {$desc} repository...\n";
|
|
||||||
$control->launchDaemon(
|
|
||||||
'PhabricatorRepositoryMercurialCommitDiscoveryDaemon',
|
|
||||||
array(
|
|
||||||
$phid,
|
|
||||||
));
|
|
||||||
break;
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
echo "Launching CommitTask daemon...\n";
|
|
||||||
$control->launchDaemon(
|
|
||||||
'PhabricatorRepositoryCommitTaskDaemon',
|
|
||||||
array());
|
|
||||||
|
|
||||||
echo "Done.\n";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
will_launch($control);
|
||||||
|
|
||||||
|
echo "Launching PullLocal daemon in master mode...\n";
|
||||||
|
$control->launchDaemon(
|
||||||
|
'PhabricatorRepositoryPullLocalDaemon',
|
||||||
|
array());
|
||||||
|
|
||||||
|
echo "Launching CommitTask daemon...\n";
|
||||||
|
$control->launchDaemon(
|
||||||
|
'PhabricatorRepositoryCommitTaskDaemon',
|
||||||
|
array());
|
||||||
|
|
||||||
|
echo "NOTE: Make sure you run some taskmaster daemons too, e.g. ".
|
||||||
|
"with 'phd launch 4 taskmaster'.\n";
|
||||||
|
|
||||||
|
echo "Done.\n";
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case 'launch':
|
case 'launch':
|
||||||
|
|
51
scripts/repository/discover.php
Executable file
51
scripts/repository/discover.php
Executable file
|
@ -0,0 +1,51 @@
|
||||||
|
#!/usr/bin/env php
|
||||||
|
<?php
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Copyright 2012 Facebook, Inc.
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
$root = dirname(dirname(dirname(__FILE__)));
|
||||||
|
require_once $root.'/scripts/__init_script__.php';
|
||||||
|
|
||||||
|
$args = new PhutilArgumentParser($argv);
|
||||||
|
$args->setTagline('manually discover working copies');
|
||||||
|
$args->setSynopsis(<<<EOHELP
|
||||||
|
**discover.php** [__options__] __repository-callsign-or-phid ...__
|
||||||
|
Manually discover commits in working copies for the named repositories.
|
||||||
|
EOHELP
|
||||||
|
);
|
||||||
|
$args->parseStandardArguments();
|
||||||
|
$args->parse(
|
||||||
|
array(
|
||||||
|
array(
|
||||||
|
'name' => 'repositories',
|
||||||
|
'wildcard' => true,
|
||||||
|
),
|
||||||
|
));
|
||||||
|
|
||||||
|
$repo_names = $args->getArg('repositories');
|
||||||
|
if (!$repo_names) {
|
||||||
|
echo "Specify one or more repositories to pull, by callsign or PHID.\n";
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
$repos = PhabricatorRepository::loadAllByPHIDOrCallsign($repo_names);
|
||||||
|
foreach ($repos as $repo) {
|
||||||
|
$callsign = $repo->getCallsign();
|
||||||
|
echo "Discovering '{$callsign}'...\n";
|
||||||
|
PhabricatorRepositoryPullLocalDaemon::discoverRepository($repo);
|
||||||
|
}
|
||||||
|
echo "Done.\n";
|
|
@ -840,7 +840,6 @@ phutil_register_library_map(array(
|
||||||
'PhabricatorRepositoryCommit' => 'applications/repository/storage/commit',
|
'PhabricatorRepositoryCommit' => 'applications/repository/storage/commit',
|
||||||
'PhabricatorRepositoryCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/base',
|
'PhabricatorRepositoryCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/base',
|
||||||
'PhabricatorRepositoryCommitData' => 'applications/repository/storage/commitdata',
|
'PhabricatorRepositoryCommitData' => 'applications/repository/storage/commitdata',
|
||||||
'PhabricatorRepositoryCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/base',
|
|
||||||
'PhabricatorRepositoryCommitHeraldWorker' => 'applications/repository/worker/herald',
|
'PhabricatorRepositoryCommitHeraldWorker' => 'applications/repository/worker/herald',
|
||||||
'PhabricatorRepositoryCommitMessageDetailParser' => 'applications/repository/parser/base',
|
'PhabricatorRepositoryCommitMessageDetailParser' => 'applications/repository/parser/base',
|
||||||
'PhabricatorRepositoryCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/base',
|
'PhabricatorRepositoryCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/base',
|
||||||
|
@ -850,22 +849,18 @@ phutil_register_library_map(array(
|
||||||
'PhabricatorRepositoryController' => 'applications/repository/controller/base',
|
'PhabricatorRepositoryController' => 'applications/repository/controller/base',
|
||||||
'PhabricatorRepositoryCreateController' => 'applications/repository/controller/create',
|
'PhabricatorRepositoryCreateController' => 'applications/repository/controller/create',
|
||||||
'PhabricatorRepositoryDAO' => 'applications/repository/storage/base',
|
'PhabricatorRepositoryDAO' => 'applications/repository/storage/base',
|
||||||
'PhabricatorRepositoryDaemon' => 'applications/repository/daemon/base',
|
|
||||||
'PhabricatorRepositoryDefaultCommitMessageDetailParser' => 'applications/repository/parser/default',
|
'PhabricatorRepositoryDefaultCommitMessageDetailParser' => 'applications/repository/parser/default',
|
||||||
'PhabricatorRepositoryDeleteController' => 'applications/repository/controller/delete',
|
'PhabricatorRepositoryDeleteController' => 'applications/repository/controller/delete',
|
||||||
'PhabricatorRepositoryEditController' => 'applications/repository/controller/edit',
|
'PhabricatorRepositoryEditController' => 'applications/repository/controller/edit',
|
||||||
'PhabricatorRepositoryGitCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/git',
|
'PhabricatorRepositoryGitCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/git',
|
||||||
'PhabricatorRepositoryGitCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/git',
|
|
||||||
'PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase' => 'applications/repository/daemon/commitdiscovery/git/__tests__',
|
|
||||||
'PhabricatorRepositoryGitCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/git',
|
'PhabricatorRepositoryGitCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/git',
|
||||||
'PhabricatorRepositoryListController' => 'applications/repository/controller/list',
|
'PhabricatorRepositoryListController' => 'applications/repository/controller/list',
|
||||||
'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/mercurial',
|
'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/mercurial',
|
||||||
'PhabricatorRepositoryMercurialCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/mercurial',
|
|
||||||
'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/mercurial',
|
'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/mercurial',
|
||||||
'PhabricatorRepositoryPullLocalDaemon' => 'applications/repository/daemon/pulllocal',
|
'PhabricatorRepositoryPullLocalDaemon' => 'applications/repository/daemon/pulllocal',
|
||||||
|
'PhabricatorRepositoryPullLocalDaemonTestCase' => 'applications/repository/daemon/pulllocal/__tests__',
|
||||||
'PhabricatorRepositoryShortcut' => 'applications/repository/storage/shortcut',
|
'PhabricatorRepositoryShortcut' => 'applications/repository/storage/shortcut',
|
||||||
'PhabricatorRepositorySvnCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/svn',
|
'PhabricatorRepositorySvnCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/svn',
|
||||||
'PhabricatorRepositorySvnCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/svn',
|
|
||||||
'PhabricatorRepositorySvnCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/svn',
|
'PhabricatorRepositorySvnCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/svn',
|
||||||
'PhabricatorRepositorySymbol' => 'applications/repository/storage/symbol',
|
'PhabricatorRepositorySymbol' => 'applications/repository/storage/symbol',
|
||||||
'PhabricatorRepositoryTestCase' => 'applications/repository/storage/repository/__tests__',
|
'PhabricatorRepositoryTestCase' => 'applications/repository/storage/repository/__tests__',
|
||||||
|
@ -1735,31 +1730,26 @@ phutil_register_library_map(array(
|
||||||
'PhabricatorRepositoryCommit' => 'PhabricatorRepositoryDAO',
|
'PhabricatorRepositoryCommit' => 'PhabricatorRepositoryDAO',
|
||||||
'PhabricatorRepositoryCommitChangeParserWorker' => 'PhabricatorRepositoryCommitParserWorker',
|
'PhabricatorRepositoryCommitChangeParserWorker' => 'PhabricatorRepositoryCommitParserWorker',
|
||||||
'PhabricatorRepositoryCommitData' => 'PhabricatorRepositoryDAO',
|
'PhabricatorRepositoryCommitData' => 'PhabricatorRepositoryDAO',
|
||||||
'PhabricatorRepositoryCommitDiscoveryDaemon' => 'PhabricatorRepositoryDaemon',
|
|
||||||
'PhabricatorRepositoryCommitHeraldWorker' => 'PhabricatorRepositoryCommitParserWorker',
|
'PhabricatorRepositoryCommitHeraldWorker' => 'PhabricatorRepositoryCommitParserWorker',
|
||||||
'PhabricatorRepositoryCommitMessageParserWorker' => 'PhabricatorRepositoryCommitParserWorker',
|
'PhabricatorRepositoryCommitMessageParserWorker' => 'PhabricatorRepositoryCommitParserWorker',
|
||||||
'PhabricatorRepositoryCommitOwnersWorker' => 'PhabricatorRepositoryCommitParserWorker',
|
'PhabricatorRepositoryCommitOwnersWorker' => 'PhabricatorRepositoryCommitParserWorker',
|
||||||
'PhabricatorRepositoryCommitParserWorker' => 'PhabricatorWorker',
|
'PhabricatorRepositoryCommitParserWorker' => 'PhabricatorWorker',
|
||||||
'PhabricatorRepositoryCommitTaskDaemon' => 'PhabricatorRepositoryDaemon',
|
'PhabricatorRepositoryCommitTaskDaemon' => 'PhabricatorDaemon',
|
||||||
'PhabricatorRepositoryController' => 'PhabricatorController',
|
'PhabricatorRepositoryController' => 'PhabricatorController',
|
||||||
'PhabricatorRepositoryCreateController' => 'PhabricatorRepositoryController',
|
'PhabricatorRepositoryCreateController' => 'PhabricatorRepositoryController',
|
||||||
'PhabricatorRepositoryDAO' => 'PhabricatorLiskDAO',
|
'PhabricatorRepositoryDAO' => 'PhabricatorLiskDAO',
|
||||||
'PhabricatorRepositoryDaemon' => 'PhabricatorDaemon',
|
|
||||||
'PhabricatorRepositoryDefaultCommitMessageDetailParser' => 'PhabricatorRepositoryCommitMessageDetailParser',
|
'PhabricatorRepositoryDefaultCommitMessageDetailParser' => 'PhabricatorRepositoryCommitMessageDetailParser',
|
||||||
'PhabricatorRepositoryDeleteController' => 'PhabricatorRepositoryController',
|
'PhabricatorRepositoryDeleteController' => 'PhabricatorRepositoryController',
|
||||||
'PhabricatorRepositoryEditController' => 'PhabricatorRepositoryController',
|
'PhabricatorRepositoryEditController' => 'PhabricatorRepositoryController',
|
||||||
'PhabricatorRepositoryGitCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
|
'PhabricatorRepositoryGitCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
|
||||||
'PhabricatorRepositoryGitCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
|
|
||||||
'PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase' => 'PhabricatorTestCase',
|
|
||||||
'PhabricatorRepositoryGitCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
|
'PhabricatorRepositoryGitCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
|
||||||
'PhabricatorRepositoryListController' => 'PhabricatorRepositoryController',
|
'PhabricatorRepositoryListController' => 'PhabricatorRepositoryController',
|
||||||
'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
|
'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
|
||||||
'PhabricatorRepositoryMercurialCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
|
|
||||||
'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
|
'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
|
||||||
'PhabricatorRepositoryPullLocalDaemon' => 'PhabricatorDaemon',
|
'PhabricatorRepositoryPullLocalDaemon' => 'PhabricatorDaemon',
|
||||||
|
'PhabricatorRepositoryPullLocalDaemonTestCase' => 'PhabricatorTestCase',
|
||||||
'PhabricatorRepositoryShortcut' => 'PhabricatorRepositoryDAO',
|
'PhabricatorRepositoryShortcut' => 'PhabricatorRepositoryDAO',
|
||||||
'PhabricatorRepositorySvnCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
|
'PhabricatorRepositorySvnCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
|
||||||
'PhabricatorRepositorySvnCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
|
|
||||||
'PhabricatorRepositorySvnCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
|
'PhabricatorRepositorySvnCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
|
||||||
'PhabricatorRepositorySymbol' => 'PhabricatorRepositoryDAO',
|
'PhabricatorRepositorySymbol' => 'PhabricatorRepositoryDAO',
|
||||||
'PhabricatorRepositoryTestCase' => 'PhabricatorTestCase',
|
'PhabricatorRepositoryTestCase' => 'PhabricatorTestCase',
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
<?php
|
<?php
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Copyright 2011 Facebook, Inc.
|
* Copyright 2012 Facebook, Inc.
|
||||||
*
|
*
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
* you may not use this file except in compliance with the License.
|
* you may not use this file except in compliance with the License.
|
||||||
|
@ -33,7 +33,11 @@ final class DiffusionSvnHistoryQuery extends DiffusionHistoryQuery {
|
||||||
PhabricatorRepository::TABLE_PATH,
|
PhabricatorRepository::TABLE_PATH,
|
||||||
array(md5('/'.trim($path, '/'))));
|
array(md5('/'.trim($path, '/'))));
|
||||||
$paths = ipull($paths, 'id', 'path');
|
$paths = ipull($paths, 'id', 'path');
|
||||||
$path_id = $paths['/'.trim($path, '/')];
|
$path_id = idx($paths, '/'.trim($path, '/'));
|
||||||
|
|
||||||
|
if (!$path_id) {
|
||||||
|
return array();
|
||||||
|
}
|
||||||
|
|
||||||
$filter_query = '';
|
$filter_query = '';
|
||||||
if ($this->needDirectChanges) {
|
if ($this->needDirectChanges) {
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
<?php
|
<?php
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Copyright 2011 Facebook, Inc.
|
* Copyright 2012 Facebook, Inc.
|
||||||
*
|
*
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
* you may not use this file except in compliance with the License.
|
* you may not use this file except in compliance with the License.
|
||||||
|
@ -32,8 +32,43 @@ abstract class PhabricatorRepositoryController extends PhabricatorController {
|
||||||
$page->setGlyph("rX");
|
$page->setGlyph("rX");
|
||||||
$page->appendChild($view);
|
$page->appendChild($view);
|
||||||
|
|
||||||
|
|
||||||
$response = new AphrontWebpageResponse();
|
$response = new AphrontWebpageResponse();
|
||||||
return $response->setContent($page->render());
|
return $response->setContent($page->render());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private function isPullDaemonRunningOnThisMachine() {
|
||||||
|
|
||||||
|
// This is sort of hacky, but should probably work.
|
||||||
|
|
||||||
|
list($stdout) = execx('ps auxwww');
|
||||||
|
return preg_match('/PhabricatorRepositoryPullLocalDaemon/', $stdout);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function renderDaemonNotice() {
|
||||||
|
$daemon_running = $this->isPullDaemonRunningOnThisMachine();
|
||||||
|
if ($daemon_running) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
$documentation = phutil_render_tag(
|
||||||
|
'a',
|
||||||
|
array(
|
||||||
|
'href' => PhabricatorEnv::getDoclink(
|
||||||
|
'article/Diffusion_User_Guide.html'),
|
||||||
|
),
|
||||||
|
'Diffusion User Guide');
|
||||||
|
|
||||||
|
$view = new AphrontErrorView();
|
||||||
|
$view->setSeverity(AphrontErrorView::SEVERITY_WARNING);
|
||||||
|
$view->setTitle('Repository Daemon Not Running');
|
||||||
|
$view->appendChild(
|
||||||
|
"<p>The repository daemon is not running on this machine. Without this ".
|
||||||
|
"daemon, Phabricator will not be able to import or update repositories. ".
|
||||||
|
"For instructions on starting the daemon, see ".
|
||||||
|
"<strong>{$documentation}</strong>.</p>");
|
||||||
|
|
||||||
|
return $view;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,7 +8,11 @@
|
||||||
|
|
||||||
phutil_require_module('phabricator', 'aphront/response/webpage');
|
phutil_require_module('phabricator', 'aphront/response/webpage');
|
||||||
phutil_require_module('phabricator', 'applications/base/controller/base');
|
phutil_require_module('phabricator', 'applications/base/controller/base');
|
||||||
|
phutil_require_module('phabricator', 'infrastructure/env');
|
||||||
|
phutil_require_module('phabricator', 'view/form/error');
|
||||||
|
|
||||||
|
phutil_require_module('phutil', 'future/exec');
|
||||||
|
phutil_require_module('phutil', 'markup');
|
||||||
phutil_require_module('phutil', 'utils');
|
phutil_require_module('phutil', 'utils');
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -63,6 +63,8 @@ final class PhabricatorRepositoryEditController
|
||||||
phutil_escape_html($name)));
|
phutil_escape_html($name)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$nav->appendChild($this->renderDaemonNotice());
|
||||||
|
|
||||||
$this->sideNav = $nav;
|
$this->sideNav = $nav;
|
||||||
|
|
||||||
switch ($this->view) {
|
switch ($this->view) {
|
||||||
|
@ -345,9 +347,7 @@ final class PhabricatorRepositoryEditController
|
||||||
$error_view = new AphrontErrorView();
|
$error_view = new AphrontErrorView();
|
||||||
$error_view->setSeverity(AphrontErrorView::SEVERITY_NOTICE);
|
$error_view->setSeverity(AphrontErrorView::SEVERITY_NOTICE);
|
||||||
$error_view->setTitle('Changes Saved');
|
$error_view->setTitle('Changes Saved');
|
||||||
$error_view->appendChild(
|
$error_view->appendChild('Tracking changes were saved.');
|
||||||
'Tracking changes were saved. You may need to restart the daemon '.
|
|
||||||
'before changes will take effect.');
|
|
||||||
} else if (!$repository->isTracked()) {
|
} else if (!$repository->isTracked()) {
|
||||||
$error_view = new AphrontErrorView();
|
$error_view = new AphrontErrorView();
|
||||||
$error_view->setSeverity(AphrontErrorView::SEVERITY_WARNING);
|
$error_view->setSeverity(AphrontErrorView::SEVERITY_WARNING);
|
||||||
|
|
|
@ -157,6 +157,7 @@ final class PhabricatorRepositoryListController
|
||||||
|
|
||||||
return $this->buildStandardPageResponse(
|
return $this->buildStandardPageResponse(
|
||||||
array(
|
array(
|
||||||
|
$this->renderDaemonNotice(),
|
||||||
$panel,
|
$panel,
|
||||||
$project_panel,
|
$project_panel,
|
||||||
),
|
),
|
||||||
|
|
|
@ -1,38 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Copyright 2011 Facebook, Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
abstract class PhabricatorRepositoryDaemon extends PhabricatorDaemon {
|
|
||||||
|
|
||||||
protected function loadRepository() {
|
|
||||||
$argv = $this->getArgv();
|
|
||||||
if (count($argv) !== 1) {
|
|
||||||
throw new Exception("No repository PHID provided!");
|
|
||||||
}
|
|
||||||
|
|
||||||
$repository = id(new PhabricatorRepository())->loadOneWhere(
|
|
||||||
'phid = %s',
|
|
||||||
$argv[0]);
|
|
||||||
|
|
||||||
if (!$repository) {
|
|
||||||
throw new Exception("No such repository exists!");
|
|
||||||
}
|
|
||||||
|
|
||||||
return $repository;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,15 +0,0 @@
|
||||||
<?php
|
|
||||||
/**
|
|
||||||
* This file is automatically generated. Lint this module to rebuild it.
|
|
||||||
* @generated
|
|
||||||
*/
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
phutil_require_module('phabricator', 'applications/repository/storage/repository');
|
|
||||||
phutil_require_module('phabricator', 'infrastructure/daemon/base');
|
|
||||||
|
|
||||||
phutil_require_module('phutil', 'utils');
|
|
||||||
|
|
||||||
|
|
||||||
phutil_require_source('PhabricatorRepositoryDaemon.php');
|
|
|
@ -1,113 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Copyright 2012 Facebook, Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
abstract class PhabricatorRepositoryCommitDiscoveryDaemon
|
|
||||||
extends PhabricatorRepositoryDaemon {
|
|
||||||
|
|
||||||
private $repository;
|
|
||||||
private $commitCache = array();
|
|
||||||
|
|
||||||
final protected function getRepository() {
|
|
||||||
return $this->repository;
|
|
||||||
}
|
|
||||||
|
|
||||||
final public function run() {
|
|
||||||
while (true) {
|
|
||||||
// Reload the repository every time to pick up changes from the web
|
|
||||||
// console.
|
|
||||||
$this->repository = $this->loadRepository();
|
|
||||||
$this->discoverCommits();
|
|
||||||
|
|
||||||
$sleep = max(2, $this->getRepository()->getDetail('pull-frequency'));
|
|
||||||
$this->sleep($sleep);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
final public function runOnce() {
|
|
||||||
$this->repository = $this->loadRepository();
|
|
||||||
$this->discoverCommits();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function isKnownCommit($target) {
|
|
||||||
if (isset($this->commitCache[$target])) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
|
|
||||||
'repositoryID = %s AND commitIdentifier = %s',
|
|
||||||
$this->getRepository()->getID(),
|
|
||||||
$target);
|
|
||||||
|
|
||||||
if (!$commit) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->commitCache[$target] = true;
|
|
||||||
while (count($this->commitCache) > 64) {
|
|
||||||
array_shift($this->commitCache);
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function recordCommit($commit_identifier, $epoch) {
|
|
||||||
$repository = $this->getRepository();
|
|
||||||
|
|
||||||
$commit = new PhabricatorRepositoryCommit();
|
|
||||||
$commit->setRepositoryID($repository->getID());
|
|
||||||
$commit->setCommitIdentifier($commit_identifier);
|
|
||||||
$commit->setEpoch($epoch);
|
|
||||||
|
|
||||||
try {
|
|
||||||
$commit->save();
|
|
||||||
$event = new PhabricatorTimelineEvent(
|
|
||||||
'cmit',
|
|
||||||
array(
|
|
||||||
'id' => $commit->getID(),
|
|
||||||
));
|
|
||||||
$event->recordEvent();
|
|
||||||
|
|
||||||
queryfx(
|
|
||||||
$repository->establishConnection('w'),
|
|
||||||
'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
|
|
||||||
VALUES (%d, 1, %d, %d)
|
|
||||||
ON DUPLICATE KEY UPDATE
|
|
||||||
size = size + 1,
|
|
||||||
lastCommitID =
|
|
||||||
IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID),
|
|
||||||
epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)',
|
|
||||||
PhabricatorRepository::TABLE_SUMMARY,
|
|
||||||
$repository->getID(),
|
|
||||||
$commit->getID(),
|
|
||||||
$epoch);
|
|
||||||
|
|
||||||
$this->commitCache[$commit_identifier] = true;
|
|
||||||
} catch (AphrontQueryDuplicateKeyException $ex) {
|
|
||||||
// Ignore. This can happen because we discover the same new commit
|
|
||||||
// more than once when looking at history, or because of races or
|
|
||||||
// data inconsistency or cosmic radiation; in any case, we're still
|
|
||||||
// in a good state if we ignore the failure.
|
|
||||||
$this->commitCache[$commit_identifier] = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->stillWorking();
|
|
||||||
}
|
|
||||||
|
|
||||||
abstract protected function discoverCommits();
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,18 +0,0 @@
|
||||||
<?php
|
|
||||||
/**
|
|
||||||
* This file is automatically generated. Lint this module to rebuild it.
|
|
||||||
* @generated
|
|
||||||
*/
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
phutil_require_module('phabricator', 'applications/repository/daemon/base');
|
|
||||||
phutil_require_module('phabricator', 'applications/repository/storage/commit');
|
|
||||||
phutil_require_module('phabricator', 'applications/repository/storage/repository');
|
|
||||||
phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/event');
|
|
||||||
phutil_require_module('phabricator', 'storage/queryfx');
|
|
||||||
|
|
||||||
phutil_require_module('phutil', 'utils');
|
|
||||||
|
|
||||||
|
|
||||||
phutil_require_source('PhabricatorRepositoryCommitDiscoveryDaemon.php');
|
|
|
@ -1,162 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Copyright 2012 Facebook, Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
final class PhabricatorRepositoryGitCommitDiscoveryDaemon
|
|
||||||
extends PhabricatorRepositoryCommitDiscoveryDaemon {
|
|
||||||
|
|
||||||
protected function discoverCommits() {
|
|
||||||
// NOTE: PhabricatorRepositoryGitFetchDaemon does the actual pulls, this
|
|
||||||
// just parses HEAD.
|
|
||||||
|
|
||||||
$repository = $this->getRepository();
|
|
||||||
|
|
||||||
$vcs = $repository->getVersionControlSystem();
|
|
||||||
if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_GIT) {
|
|
||||||
throw new Exception("Repository is not a git repository.");
|
|
||||||
}
|
|
||||||
|
|
||||||
list($remotes) = $repository->execxLocalCommand(
|
|
||||||
'remote show -n origin');
|
|
||||||
|
|
||||||
$matches = null;
|
|
||||||
if (!preg_match('/^\s*Fetch URL:\s*(.*?)\s*$/m', $remotes, $matches)) {
|
|
||||||
throw new Exception(
|
|
||||||
"Expected 'Fetch URL' in 'git remote show -n origin'.");
|
|
||||||
}
|
|
||||||
|
|
||||||
self::verifySameGitOrigin(
|
|
||||||
$matches[1],
|
|
||||||
$repository->getRemoteURI(),
|
|
||||||
$repository->getLocalPath());
|
|
||||||
|
|
||||||
list($stdout) = $repository->execxLocalCommand(
|
|
||||||
'branch -r --verbose --no-abbrev');
|
|
||||||
|
|
||||||
$branches = DiffusionGitBranchQuery::parseGitRemoteBranchOutput(
|
|
||||||
$stdout,
|
|
||||||
$only_this_remote = DiffusionBranchInformation::DEFAULT_GIT_REMOTE);
|
|
||||||
|
|
||||||
$got_something = false;
|
|
||||||
$tracked_something = false;
|
|
||||||
foreach ($branches as $name => $commit) {
|
|
||||||
if (!$repository->shouldTrackBranch($name)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
$tracked_something = true;
|
|
||||||
|
|
||||||
if ($this->isKnownCommit($commit)) {
|
|
||||||
continue;
|
|
||||||
} else {
|
|
||||||
$this->discoverCommit($commit);
|
|
||||||
$got_something = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!$tracked_something) {
|
|
||||||
$repo_name = $repository->getName();
|
|
||||||
$repo_callsign = $repository->getCallsign();
|
|
||||||
throw new Exception(
|
|
||||||
"Repository r{$repo_callsign} '{$repo_name}' has no tracked branches! ".
|
|
||||||
"Verify that your branch filtering settings are correct.");
|
|
||||||
}
|
|
||||||
|
|
||||||
return $got_something;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function discoverCommit($commit) {
|
|
||||||
$discover = array();
|
|
||||||
$insert = array();
|
|
||||||
|
|
||||||
$repository = $this->getRepository();
|
|
||||||
|
|
||||||
$discover[] = $commit;
|
|
||||||
$insert[] = $commit;
|
|
||||||
|
|
||||||
$seen_parent = array();
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
$target = array_pop($discover);
|
|
||||||
list($parents) = $repository->execxLocalCommand(
|
|
||||||
'log -n1 --pretty="%%P" %s',
|
|
||||||
$target);
|
|
||||||
$parents = array_filter(explode(' ', trim($parents)));
|
|
||||||
foreach ($parents as $parent) {
|
|
||||||
if (isset($seen_parent[$parent])) {
|
|
||||||
// We end up in a loop here somehow when we parse Arcanist if we
|
|
||||||
// don't do this. TODO: Figure out why and draw a pretty diagram
|
|
||||||
// since it's not evident how parsing a DAG with this causes the
|
|
||||||
// loop to stop terminating.
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$seen_parent[$parent] = true;
|
|
||||||
if (!$this->isKnownCommit($parent)) {
|
|
||||||
$discover[] = $parent;
|
|
||||||
$insert[] = $parent;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (empty($discover)) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
$this->stillWorking();
|
|
||||||
}
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
$target = array_pop($insert);
|
|
||||||
list($epoch) = $repository->execxLocalCommand(
|
|
||||||
'log -n1 --pretty="%%ct" %s',
|
|
||||||
$target);
|
|
||||||
$epoch = trim($epoch);
|
|
||||||
|
|
||||||
$this->recordCommit($target, $epoch);
|
|
||||||
|
|
||||||
if (empty($insert)) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static function verifySameGitOrigin($remote, $expect, $where) {
|
|
||||||
$remote_uri = PhabricatorRepository::newPhutilURIFromGitURI($remote);
|
|
||||||
$expect_uri = PhabricatorRepository::newPhutilURIFromGitURI($expect);
|
|
||||||
|
|
||||||
$remote_path = $remote_uri->getPath();
|
|
||||||
$expect_path = $expect_uri->getPath();
|
|
||||||
|
|
||||||
$remote_match = self::normalizeGitPath($remote_path);
|
|
||||||
$expect_match = self::normalizeGitPath($expect_path);
|
|
||||||
|
|
||||||
if ($remote_match != $expect_match) {
|
|
||||||
throw new Exception(
|
|
||||||
"Working copy at '{$where}' has a mismatched origin URL. It has ".
|
|
||||||
"origin URL '{$remote}' (with remote path '{$remote_path}'), but the ".
|
|
||||||
"configured URL '{$expect}' (with remote path '{$expect_path}') is ".
|
|
||||||
"expected. Refusing to proceed because this may indicate that the ".
|
|
||||||
"working copy is actually some other repository.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static function normalizeGitPath($path) {
|
|
||||||
// Strip away trailing "/" and ".git", so similar paths correctly match.
|
|
||||||
|
|
||||||
$path = rtrim($path, '/');
|
|
||||||
$path = preg_replace('/\.git$/', '', $path);
|
|
||||||
return $path;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,16 +0,0 @@
|
||||||
<?php
|
|
||||||
/**
|
|
||||||
* This file is automatically generated. Lint this module to rebuild it.
|
|
||||||
* @generated
|
|
||||||
*/
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
phutil_require_module('phabricator', 'applications/diffusion/data/branch');
|
|
||||||
phutil_require_module('phabricator', 'applications/diffusion/query/branch/git');
|
|
||||||
phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
|
|
||||||
phutil_require_module('phabricator', 'applications/repository/daemon/commitdiscovery/base');
|
|
||||||
phutil_require_module('phabricator', 'applications/repository/storage/repository');
|
|
||||||
|
|
||||||
|
|
||||||
phutil_require_source('PhabricatorRepositoryGitCommitDiscoveryDaemon.php');
|
|
|
@ -1,137 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Copyright 2012 Facebook, Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
final class PhabricatorRepositoryMercurialCommitDiscoveryDaemon
|
|
||||||
extends PhabricatorRepositoryCommitDiscoveryDaemon {
|
|
||||||
|
|
||||||
protected function discoverCommits() {
|
|
||||||
$repository = $this->getRepository();
|
|
||||||
|
|
||||||
$vcs = $repository->getVersionControlSystem();
|
|
||||||
if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL) {
|
|
||||||
throw new Exception("Repository is not a Mercurial repository.");
|
|
||||||
}
|
|
||||||
|
|
||||||
$repository_phid = $repository->getPHID();
|
|
||||||
|
|
||||||
list($stdout) = $repository->execxLocalCommand('branches');
|
|
||||||
|
|
||||||
$branches = ArcanistMercurialParser::parseMercurialBranches($stdout);
|
|
||||||
$got_something = false;
|
|
||||||
foreach ($branches as $name => $branch) {
|
|
||||||
$commit = $branch['rev'];
|
|
||||||
$commit = $this->getFullHash($commit);
|
|
||||||
if ($this->isKnownCommit($commit)) {
|
|
||||||
continue;
|
|
||||||
} else {
|
|
||||||
$this->discoverCommit($commit);
|
|
||||||
$got_something = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return $got_something;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function getFullHash($commit) {
|
|
||||||
|
|
||||||
// NOTE: Mercurial shortens hashes to 12 characters by default. This
|
|
||||||
// implies collisions with as few as a few million commits. The
|
|
||||||
// documentation sensibly advises "Do not use short-form IDs for
|
|
||||||
// long-lived representations". It then continues "You can use the
|
|
||||||
// --debug option to display the full changeset ID". What?! Yes, this
|
|
||||||
// is in fact the only way to turn on full hashes, and the hg source
|
|
||||||
// code is littered with "hexfn = ui.debugflag and hex or short" and
|
|
||||||
// similar. There is no more-selective flag or config option.
|
|
||||||
//
|
|
||||||
// Unfortunately, "hg --debug" turns on tons of other extra output,
|
|
||||||
// including full commit messages in "hg log" and "hg parents" (which
|
|
||||||
// ignore --style); this renders them unparseable. So we have to use
|
|
||||||
// "hg id" to convert short hashes into full hashes. See:
|
|
||||||
//
|
|
||||||
// <http://mercurial.selenic.com/wiki/ChangeSetID>
|
|
||||||
//
|
|
||||||
// Of course, this means that if there are collisions we will break here
|
|
||||||
// (the short commit identifier won't be unambiguous) but maybe Mercurial
|
|
||||||
// will have a --full-hashes flag or something by then and we can fix it
|
|
||||||
// properly. Until we run into that, this allows us to store data in the
|
|
||||||
// right format so when we eventually encounter this we won't have to
|
|
||||||
// reparse every Mercurial repository.
|
|
||||||
|
|
||||||
$repository = $this->getRepository();
|
|
||||||
list($stdout) = $repository->execxLocalCommand(
|
|
||||||
'id --debug -i --rev %s',
|
|
||||||
$commit);
|
|
||||||
return trim($stdout);
|
|
||||||
}
|
|
||||||
|
|
||||||
private function discoverCommit($commit) {
|
|
||||||
$discover = array();
|
|
||||||
$insert = array();
|
|
||||||
|
|
||||||
$repository = $this->getRepository();
|
|
||||||
|
|
||||||
$discover[] = $commit;
|
|
||||||
$insert[] = $commit;
|
|
||||||
|
|
||||||
$seen_parent = array();
|
|
||||||
|
|
||||||
// For all the new commits at the branch heads, walk backward until we find
|
|
||||||
// only commits we've aleady seen.
|
|
||||||
while (true) {
|
|
||||||
$target = array_pop($discover);
|
|
||||||
list($stdout) = $repository->execxLocalCommand(
|
|
||||||
'parents --style default --rev %s',
|
|
||||||
$target);
|
|
||||||
$parents = ArcanistMercurialParser::parseMercurialLog($stdout);
|
|
||||||
if ($parents) {
|
|
||||||
foreach ($parents as $parent) {
|
|
||||||
$parent_commit = $parent['rev'];
|
|
||||||
$parent_commit = $this->getFullHash($parent_commit);
|
|
||||||
if (isset($seen_parent[$parent_commit])) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$seen_parent[$parent_commit] = true;
|
|
||||||
if (!$this->isKnownCommit($parent_commit)) {
|
|
||||||
$discover[] = $parent_commit;
|
|
||||||
$insert[] = $parent_commit;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (empty($discover)) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
$this->stillWorking();
|
|
||||||
}
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
$target = array_pop($insert);
|
|
||||||
list($stdout) = $repository->execxLocalCommand(
|
|
||||||
'log --rev %s --template %s',
|
|
||||||
$target,
|
|
||||||
'{date|rfc822date}');
|
|
||||||
$epoch = strtotime($stdout);
|
|
||||||
|
|
||||||
$this->recordCommit($target, $epoch);
|
|
||||||
|
|
||||||
if (empty($insert)) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,15 +0,0 @@
|
||||||
<?php
|
|
||||||
/**
|
|
||||||
* This file is automatically generated. Lint this module to rebuild it.
|
|
||||||
* @generated
|
|
||||||
*/
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
phutil_require_module('arcanist', 'repository/parser/mercurial');
|
|
||||||
|
|
||||||
phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
|
|
||||||
phutil_require_module('phabricator', 'applications/repository/daemon/commitdiscovery/base');
|
|
||||||
|
|
||||||
|
|
||||||
phutil_require_source('PhabricatorRepositoryMercurialCommitDiscoveryDaemon.php');
|
|
|
@ -1,123 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Copyright 2012 Facebook, Inc.
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
final class PhabricatorRepositorySvnCommitDiscoveryDaemon
|
|
||||||
extends PhabricatorRepositoryCommitDiscoveryDaemon {
|
|
||||||
|
|
||||||
protected function discoverCommits() {
|
|
||||||
$repository = $this->getRepository();
|
|
||||||
|
|
||||||
$vcs = $repository->getVersionControlSystem();
|
|
||||||
if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_SVN) {
|
|
||||||
throw new Exception("Repository is not a svn repository.");
|
|
||||||
}
|
|
||||||
|
|
||||||
$uri = $this->getBaseSVNLogURI();
|
|
||||||
list($xml) = $repository->execxRemoteCommand(
|
|
||||||
'log --xml --quiet --limit 1 %s@HEAD',
|
|
||||||
$uri);
|
|
||||||
|
|
||||||
$results = $this->parseSVNLogXML($xml);
|
|
||||||
$commit = head_key($results);
|
|
||||||
$epoch = head($results);
|
|
||||||
|
|
||||||
if ($this->isKnownCommit($commit)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->discoverCommit($commit, $epoch);
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function discoverCommit($commit, $epoch) {
|
|
||||||
$uri = $this->getBaseSVNLogURI();
|
|
||||||
$repository = $this->getRepository();
|
|
||||||
|
|
||||||
$discover = array(
|
|
||||||
$commit => $epoch,
|
|
||||||
);
|
|
||||||
$upper_bound = $commit;
|
|
||||||
|
|
||||||
$limit = 1;
|
|
||||||
while ($upper_bound > 1 && !$this->isKnownCommit($upper_bound)) {
|
|
||||||
// Find all the unknown commits on this path. Note that we permit
|
|
||||||
// importing an SVN subdirectory rather than the entire repository, so
|
|
||||||
// commits may be nonsequential.
|
|
||||||
list($err, $xml, $stderr) = $repository->execRemoteCommand(
|
|
||||||
' log --xml --quiet --limit %d %s@%d',
|
|
||||||
$limit,
|
|
||||||
$uri,
|
|
||||||
$upper_bound - 1);
|
|
||||||
if ($err) {
|
|
||||||
if (preg_match('/(path|File) not found/', $stderr)) {
|
|
||||||
// We've gone all the way back through history and this path was not
|
|
||||||
// affected by earlier commits.
|
|
||||||
break;
|
|
||||||
} else {
|
|
||||||
throw new Exception("svn log error #{$err}: {$stderr}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$discover += $this->parseSVNLogXML($xml);
|
|
||||||
|
|
||||||
$upper_bound = min(array_keys($discover));
|
|
||||||
|
|
||||||
// Discover 2, 4, 8, ... 256 logs at a time. This allows us to initially
|
|
||||||
// import large repositories fairly quickly, while pulling only as much
|
|
||||||
// data as we need in the common case (when we've already imported the
|
|
||||||
// repository and are just grabbing one commit at a time).
|
|
||||||
$limit = min($limit * 2, 256);
|
|
||||||
}
|
|
||||||
|
|
||||||
// NOTE: We do writes only after discovering all the commits so that we're
|
|
||||||
// never left in a state where we've missed commits -- if the discovery
|
|
||||||
// script terminates it can always resume and restore the import to a good
|
|
||||||
// state. This is also why we sort the discovered commits so we can do
|
|
||||||
// writes forward from the smallest one.
|
|
||||||
|
|
||||||
ksort($discover);
|
|
||||||
foreach ($discover as $commit => $epoch) {
|
|
||||||
$this->recordCommit($commit, $epoch);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function parseSVNLogXML($xml) {
|
|
||||||
$xml = phutil_utf8ize($xml);
|
|
||||||
|
|
||||||
$result = array();
|
|
||||||
|
|
||||||
$log = new SimpleXMLElement($xml);
|
|
||||||
foreach ($log->logentry as $entry) {
|
|
||||||
$commit = (int)$entry['revision'];
|
|
||||||
$epoch = (int)strtotime((string)$entry->date[0]);
|
|
||||||
$result[$commit] = $epoch;
|
|
||||||
}
|
|
||||||
|
|
||||||
return $result;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private function getBaseSVNLogURI() {
|
|
||||||
$repository = $this->getRepository();
|
|
||||||
|
|
||||||
$uri = $repository->getDetail('remote-uri');
|
|
||||||
$subpath = $repository->getDetail('svn-subpath');
|
|
||||||
|
|
||||||
return $uri.$subpath;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,15 +0,0 @@
|
||||||
<?php
|
|
||||||
/**
|
|
||||||
* This file is automatically generated. Lint this module to rebuild it.
|
|
||||||
* @generated
|
|
||||||
*/
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
|
|
||||||
phutil_require_module('phabricator', 'applications/repository/daemon/commitdiscovery/base');
|
|
||||||
|
|
||||||
phutil_require_module('phutil', 'utils');
|
|
||||||
|
|
||||||
|
|
||||||
phutil_require_source('PhabricatorRepositorySvnCommitDiscoveryDaemon.php');
|
|
|
@ -17,7 +17,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
final class PhabricatorRepositoryCommitTaskDaemon
|
final class PhabricatorRepositoryCommitTaskDaemon
|
||||||
extends PhabricatorRepositoryDaemon {
|
extends PhabricatorDaemon {
|
||||||
|
|
||||||
final public function run() {
|
final public function run() {
|
||||||
do {
|
do {
|
||||||
|
|
|
@ -7,9 +7,9 @@
|
||||||
|
|
||||||
|
|
||||||
phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
|
phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
|
||||||
phutil_require_module('phabricator', 'applications/repository/daemon/base');
|
|
||||||
phutil_require_module('phabricator', 'applications/repository/storage/commit');
|
phutil_require_module('phabricator', 'applications/repository/storage/commit');
|
||||||
phutil_require_module('phabricator', 'applications/repository/storage/repository');
|
phutil_require_module('phabricator', 'applications/repository/storage/repository');
|
||||||
|
phutil_require_module('phabricator', 'infrastructure/daemon/base');
|
||||||
phutil_require_module('phabricator', 'infrastructure/daemon/timeline/cursor/iterator');
|
phutil_require_module('phabricator', 'infrastructure/daemon/timeline/cursor/iterator');
|
||||||
phutil_require_module('phabricator', 'infrastructure/daemon/workers/storage/task');
|
phutil_require_module('phabricator', 'infrastructure/daemon/workers/storage/task');
|
||||||
|
|
||||||
|
|
|
@ -24,7 +24,12 @@
|
||||||
* responsible for only some repositories, you can launch it with a list of
|
* responsible for only some repositories, you can launch it with a list of
|
||||||
* PHIDs or callsigns:
|
* PHIDs or callsigns:
|
||||||
*
|
*
|
||||||
* ./phd launch repositorypulllocal X Q Z
|
* ./phd launch repositorypulllocal -- X Q Z
|
||||||
|
*
|
||||||
|
* You can also launch a daemon which is responsible for all //but// one or
|
||||||
|
* more repositories:
|
||||||
|
*
|
||||||
|
* ./phd launch repositorypulllocal -- --not A --not B
|
||||||
*
|
*
|
||||||
* If you have a very large number of repositories and some aren't being pulled
|
* If you have a very large number of repositories and some aren't being pulled
|
||||||
* as frequently as you'd like, you can either change the pull frequency of
|
* as frequently as you'd like, you can either change the pull frequency of
|
||||||
|
@ -40,6 +45,8 @@
|
||||||
final class PhabricatorRepositoryPullLocalDaemon
|
final class PhabricatorRepositoryPullLocalDaemon
|
||||||
extends PhabricatorDaemon {
|
extends PhabricatorDaemon {
|
||||||
|
|
||||||
|
private static $commitCache = array();
|
||||||
|
|
||||||
|
|
||||||
/* -( Pulling Repositories )----------------------------------------------- */
|
/* -( Pulling Repositories )----------------------------------------------- */
|
||||||
|
|
||||||
|
@ -48,14 +55,45 @@ final class PhabricatorRepositoryPullLocalDaemon
|
||||||
* @task pull
|
* @task pull
|
||||||
*/
|
*/
|
||||||
public function run() {
|
public function run() {
|
||||||
|
$argv = $this->getArgv();
|
||||||
|
array_unshift($argv, __CLASS__);
|
||||||
|
$args = new PhutilArgumentParser($argv);
|
||||||
|
$args->parse(
|
||||||
|
array(
|
||||||
|
array(
|
||||||
|
'name' => 'no-discovery',
|
||||||
|
'help' => 'Pull only, without discovering commits.',
|
||||||
|
),
|
||||||
|
array(
|
||||||
|
'name' => 'not',
|
||||||
|
'param' => 'repository',
|
||||||
|
'repeat' => true,
|
||||||
|
'help' => 'Do not pull __repository__.',
|
||||||
|
),
|
||||||
|
array(
|
||||||
|
'name' => 'repositories',
|
||||||
|
'wildcard' => true,
|
||||||
|
'help' => 'Pull specific __repositories__ instead of all.',
|
||||||
|
),
|
||||||
|
));
|
||||||
|
|
||||||
|
$no_discovery = $args->getArg('no-discovery');
|
||||||
|
$repo_names = $args->getArg('repositories', array());
|
||||||
|
$exclude_names = $args->getArg('not', array());
|
||||||
|
|
||||||
// Each repository has an individual pull frequency; after we pull it,
|
// Each repository has an individual pull frequency; after we pull it,
|
||||||
// wait that long to pull it again. When we start up, try to pull everything
|
// wait that long to pull it again. When we start up, try to pull everything
|
||||||
// serially.
|
// serially.
|
||||||
$retry_after = array();
|
$retry_after = array();
|
||||||
|
|
||||||
|
$min_sleep = 15;
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
$repositories = $this->loadRepositories();
|
$repositories = $this->loadRepositories($repo_names);
|
||||||
|
if ($exclude_names) {
|
||||||
|
$exclude = $this->loadRepositories($exclude_names);
|
||||||
|
$repositories = array_diff_key($repositories, $exclude);
|
||||||
|
}
|
||||||
|
|
||||||
// Shuffle the repositories, then re-key the array since shuffle()
|
// Shuffle the repositories, then re-key the array since shuffle()
|
||||||
// discards keys. This is mostly for startup, we'll use soft priorities
|
// discards keys. This is mostly for startup, we'll use soft priorities
|
||||||
|
@ -79,22 +117,41 @@ final class PhabricatorRepositoryPullLocalDaemon
|
||||||
|
|
||||||
foreach ($repositories as $id => $repository) {
|
foreach ($repositories as $id => $repository) {
|
||||||
$after = idx($retry_after, $id, 0);
|
$after = idx($retry_after, $id, 0);
|
||||||
if ($after >= time()) {
|
if ($after > time()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
$tracked = $repository->isTracked();
|
||||||
|
if (!$tracked) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
self::pullRepository($repository);
|
self::pullRepository($repository);
|
||||||
$sleep_for = $repository->getDetail('pull-frequency', 15);
|
|
||||||
|
if (!$no_discovery) {
|
||||||
|
// TODO: It would be nice to discover only if we pulled something,
|
||||||
|
// but this isn't totally trivial.
|
||||||
|
self::discoverRepository($repository);
|
||||||
|
}
|
||||||
|
|
||||||
|
$sleep_for = $repository->getDetail('pull-frequency', $min_sleep);
|
||||||
$retry_after[$id] = time() + $sleep_for;
|
$retry_after[$id] = time() + $sleep_for;
|
||||||
} catch (Exception $ex) {
|
} catch (Exception $ex) {
|
||||||
$retry_after[$id] = time() + 15;
|
$retry_after[$id] = time() + $min_sleep;
|
||||||
phlog($ex);
|
phlog($ex);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$this->stillWorking();
|
||||||
}
|
}
|
||||||
|
|
||||||
$sleep_until = max(min($retry_after), time() + 15);
|
if ($retry_after) {
|
||||||
sleep($sleep_until - time());
|
$sleep_until = max(min($retry_after), time() + $min_sleep);
|
||||||
|
} else {
|
||||||
|
$sleep_until = time() + $min_sleep;
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->sleep($sleep_until - time());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -102,12 +159,11 @@ final class PhabricatorRepositoryPullLocalDaemon
|
||||||
/**
|
/**
|
||||||
* @task pull
|
* @task pull
|
||||||
*/
|
*/
|
||||||
protected function loadRepositories() {
|
protected function loadRepositories(array $names) {
|
||||||
$argv = $this->getArgv();
|
if (!count($names)) {
|
||||||
if (!count($argv)) {
|
|
||||||
return id(new PhabricatorRepository())->loadAll();
|
return id(new PhabricatorRepository())->loadAll();
|
||||||
} else {
|
} else {
|
||||||
return PhabricatorRepository::loadAllByPHIDOrCallsign($argv);
|
return PhabricatorRepository::loadAllByPHIDOrCallsign($names);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -116,11 +172,6 @@ final class PhabricatorRepositoryPullLocalDaemon
|
||||||
* @task pull
|
* @task pull
|
||||||
*/
|
*/
|
||||||
public static function pullRepository(PhabricatorRepository $repository) {
|
public static function pullRepository(PhabricatorRepository $repository) {
|
||||||
$tracked = $repository->isTracked();
|
|
||||||
if (!$tracked) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
$vcs = $repository->getVersionControlSystem();
|
$vcs = $repository->getVersionControlSystem();
|
||||||
|
|
||||||
$is_svn = ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_SVN);
|
$is_svn = ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_SVN);
|
||||||
|
@ -153,19 +204,126 @@ final class PhabricatorRepositoryPullLocalDaemon
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($is_git) {
|
if ($is_git) {
|
||||||
self::executeGitCreate($repository, $local_path);
|
return self::executeGitCreate($repository, $local_path);
|
||||||
} else if ($is_hg) {
|
} else if ($is_hg) {
|
||||||
self::executeHgCreate($repository, $local_path);
|
return self::executeHgCreate($repository, $local_path);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if ($is_git) {
|
if ($is_git) {
|
||||||
self::executeGitUpdate($repository, $local_path);
|
return self::executeGitUpdate($repository, $local_path);
|
||||||
} else if ($is_hg) {
|
} else if ($is_hg) {
|
||||||
self::executeHgUpdate($repository, $local_path);
|
return self::executeHgUpdate($repository, $local_path);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static function discoverRepository(PhabricatorRepository $repository) {
|
||||||
|
$vcs = $repository->getVersionControlSystem();
|
||||||
|
switch ($vcs) {
|
||||||
|
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
|
||||||
|
return self::executeGitDiscover($repository);
|
||||||
|
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
|
||||||
|
return self::executeSvnDiscover($repository);
|
||||||
|
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
|
||||||
|
return self::executeHgDiscover($repository);
|
||||||
|
default:
|
||||||
|
throw new Exception("Unknown VCS '{$vcs}'!");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static function isKnownCommit(
|
||||||
|
PhabricatorRepository $repository,
|
||||||
|
$target) {
|
||||||
|
|
||||||
|
if (self::getCache($repository, $target)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
|
||||||
|
'repositoryID = %s AND commitIdentifier = %s',
|
||||||
|
$repository->getID(),
|
||||||
|
$target);
|
||||||
|
|
||||||
|
if (!$commit) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
self::setCache($repository, $target);
|
||||||
|
while (count(self::$commitCache) > 2048) {
|
||||||
|
array_shift(self::$commitCache);
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function recordCommit(
|
||||||
|
PhabricatorRepository $repository,
|
||||||
|
$commit_identifier,
|
||||||
|
$epoch) {
|
||||||
|
|
||||||
|
$commit = new PhabricatorRepositoryCommit();
|
||||||
|
$commit->setRepositoryID($repository->getID());
|
||||||
|
$commit->setCommitIdentifier($commit_identifier);
|
||||||
|
$commit->setEpoch($epoch);
|
||||||
|
|
||||||
|
try {
|
||||||
|
$commit->save();
|
||||||
|
$event = new PhabricatorTimelineEvent(
|
||||||
|
'cmit',
|
||||||
|
array(
|
||||||
|
'id' => $commit->getID(),
|
||||||
|
));
|
||||||
|
$event->recordEvent();
|
||||||
|
|
||||||
|
queryfx(
|
||||||
|
$repository->establishConnection('w'),
|
||||||
|
'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
|
||||||
|
VALUES (%d, 1, %d, %d)
|
||||||
|
ON DUPLICATE KEY UPDATE
|
||||||
|
size = size + 1,
|
||||||
|
lastCommitID =
|
||||||
|
IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID),
|
||||||
|
epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)',
|
||||||
|
PhabricatorRepository::TABLE_SUMMARY,
|
||||||
|
$repository->getID(),
|
||||||
|
$commit->getID(),
|
||||||
|
$epoch);
|
||||||
|
|
||||||
|
self::setCache($repository, $commit_identifier);
|
||||||
|
} catch (AphrontQueryDuplicateKeyException $ex) {
|
||||||
|
// Ignore. This can happen because we discover the same new commit
|
||||||
|
// more than once when looking at history, or because of races or
|
||||||
|
// data inconsistency or cosmic radiation; in any case, we're still
|
||||||
|
// in a good state if we ignore the failure.
|
||||||
|
self::setCache($repository, $commit_identifier);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function setCache(
|
||||||
|
PhabricatorRepository $repository,
|
||||||
|
$commit_identifier) {
|
||||||
|
|
||||||
|
$key = self::getCacheKey($repository, $commit_identifier);
|
||||||
|
self::$commitCache[$key] = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function getCache(
|
||||||
|
PhabricatorRepository $repository,
|
||||||
|
$commit_identifier) {
|
||||||
|
|
||||||
|
$key = self::getCacheKey($repository, $commit_identifier);
|
||||||
|
return idx(self::$commitCache, $key, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function getCacheKey(
|
||||||
|
PhabricatorRepository $repository,
|
||||||
|
$commit_identifier) {
|
||||||
|
|
||||||
|
return $repository->getID().':'.$commit_identifier;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/* -( Git Implementation )------------------------------------------------- */
|
/* -( Git Implementation )------------------------------------------------- */
|
||||||
|
|
||||||
|
@ -249,6 +407,147 @@ final class PhabricatorRepositoryPullLocalDaemon
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @task git
|
||||||
|
*/
|
||||||
|
private static function executeGitDiscover(
|
||||||
|
PhabricatorRepository $repository) {
|
||||||
|
|
||||||
|
list($remotes) = $repository->execxLocalCommand(
|
||||||
|
'remote show -n origin');
|
||||||
|
|
||||||
|
$matches = null;
|
||||||
|
if (!preg_match('/^\s*Fetch URL:\s*(.*?)\s*$/m', $remotes, $matches)) {
|
||||||
|
throw new Exception(
|
||||||
|
"Expected 'Fetch URL' in 'git remote show -n origin'.");
|
||||||
|
}
|
||||||
|
|
||||||
|
self::executeGitverifySameOrigin(
|
||||||
|
$matches[1],
|
||||||
|
$repository->getRemoteURI(),
|
||||||
|
$repository->getLocalPath());
|
||||||
|
|
||||||
|
list($stdout) = $repository->execxLocalCommand(
|
||||||
|
'branch -r --verbose --no-abbrev');
|
||||||
|
|
||||||
|
$branches = DiffusionGitBranchQuery::parseGitRemoteBranchOutput(
|
||||||
|
$stdout,
|
||||||
|
$only_this_remote = DiffusionBranchInformation::DEFAULT_GIT_REMOTE);
|
||||||
|
|
||||||
|
$tracked_something = false;
|
||||||
|
foreach ($branches as $name => $commit) {
|
||||||
|
if (!$repository->shouldTrackBranch($name)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
$tracked_something = true;
|
||||||
|
|
||||||
|
if (self::isKnownCommit($repository, $commit)) {
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
|
self::executeGitDiscoverCommit($repository, $commit);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!$tracked_something) {
|
||||||
|
$repo_name = $repository->getName();
|
||||||
|
$repo_callsign = $repository->getCallsign();
|
||||||
|
throw new Exception(
|
||||||
|
"Repository r{$repo_callsign} '{$repo_name}' has no tracked branches! ".
|
||||||
|
"Verify that your branch filtering settings are correct.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @task git
|
||||||
|
*/
|
||||||
|
private static function executeGitDiscoverCommit(
|
||||||
|
PhabricatorRepository $repository,
|
||||||
|
$commit) {
|
||||||
|
|
||||||
|
$discover = array($commit);
|
||||||
|
$insert = array($commit);
|
||||||
|
|
||||||
|
$seen_parent = array();
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
$target = array_pop($discover);
|
||||||
|
list($parents) = $repository->execxLocalCommand(
|
||||||
|
'log -n1 --pretty="%%P" %s',
|
||||||
|
$target);
|
||||||
|
$parents = array_filter(explode(' ', trim($parents)));
|
||||||
|
foreach ($parents as $parent) {
|
||||||
|
if (isset($seen_parent[$parent])) {
|
||||||
|
// We end up in a loop here somehow when we parse Arcanist if we
|
||||||
|
// don't do this. TODO: Figure out why and draw a pretty diagram
|
||||||
|
// since it's not evident how parsing a DAG with this causes the
|
||||||
|
// loop to stop terminating.
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$seen_parent[$parent] = true;
|
||||||
|
if (!self::isKnownCommit($repository, $parent)) {
|
||||||
|
$discover[] = $parent;
|
||||||
|
$insert[] = $parent;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (empty($discover)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
$target = array_pop($insert);
|
||||||
|
list($epoch) = $repository->execxLocalCommand(
|
||||||
|
'log -n1 --pretty="%%ct" %s',
|
||||||
|
$target);
|
||||||
|
$epoch = trim($epoch);
|
||||||
|
|
||||||
|
self::recordCommit($repository, $target, $epoch);
|
||||||
|
|
||||||
|
if (empty($insert)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @task git
|
||||||
|
*/
|
||||||
|
public static function executeGitVerifySameOrigin($remote, $expect, $where) {
|
||||||
|
$remote_uri = PhabricatorRepository::newPhutilURIFromGitURI($remote);
|
||||||
|
$expect_uri = PhabricatorRepository::newPhutilURIFromGitURI($expect);
|
||||||
|
|
||||||
|
$remote_path = $remote_uri->getPath();
|
||||||
|
$expect_path = $expect_uri->getPath();
|
||||||
|
|
||||||
|
$remote_match = self::executeGitNormalizePath($remote_path);
|
||||||
|
$expect_match = self::executeGitNormalizePath($expect_path);
|
||||||
|
|
||||||
|
if ($remote_match != $expect_match) {
|
||||||
|
throw new Exception(
|
||||||
|
"Working copy at '{$where}' has a mismatched origin URL. It has ".
|
||||||
|
"origin URL '{$remote}' (with remote path '{$remote_path}'), but the ".
|
||||||
|
"configured URL '{$expect}' (with remote path '{$expect_path}') is ".
|
||||||
|
"expected. Refusing to proceed because this may indicate that the ".
|
||||||
|
"working copy is actually some other repository.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @task git
|
||||||
|
*/
|
||||||
|
private static function executeGitNormalizePath($path) {
|
||||||
|
// Strip away trailing "/" and ".git", so similar paths correctly match.
|
||||||
|
|
||||||
|
$path = rtrim($path, '/');
|
||||||
|
$path = preg_replace('/\.git$/', '', $path);
|
||||||
|
return $path;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/* -( Mercurial Implementation )------------------------------------------- */
|
/* -( Mercurial Implementation )------------------------------------------- */
|
||||||
|
|
||||||
|
|
||||||
|
@ -306,4 +605,177 @@ final class PhabricatorRepositoryPullLocalDaemon
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static function executeHgDiscover(PhabricatorRepository $repository) {
|
||||||
|
// NOTE: "--debug" gives us 40-character hashes.
|
||||||
|
list($stdout) = $repository->execxLocalCommand('--debug branches');
|
||||||
|
|
||||||
|
$branches = ArcanistMercurialParser::parseMercurialBranches($stdout);
|
||||||
|
$got_something = false;
|
||||||
|
foreach ($branches as $name => $branch) {
|
||||||
|
$commit = $branch['rev'];
|
||||||
|
if (self::isKnownCommit($repository, $commit)) {
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
|
self::executeHgDiscoverCommit($repository, $commit);
|
||||||
|
$got_something = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return $got_something;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function executeHgDiscoverCommit(
|
||||||
|
PhabricatorRepository $repository,
|
||||||
|
$commit) {
|
||||||
|
|
||||||
|
$discover = array($commit);
|
||||||
|
$insert = array($commit);
|
||||||
|
|
||||||
|
$seen_parent = array();
|
||||||
|
|
||||||
|
// For all the new commits at the branch heads, walk backward until we find
|
||||||
|
// only commits we've aleady seen.
|
||||||
|
while (true) {
|
||||||
|
$target = array_pop($discover);
|
||||||
|
list($stdout) = $repository->execxLocalCommand(
|
||||||
|
'parents --rev %s --template %s',
|
||||||
|
$target,
|
||||||
|
'{node}\n');
|
||||||
|
$parents = array_filter(explode("\n", trim($stdout)));
|
||||||
|
foreach ($parents as $parent) {
|
||||||
|
if (isset($seen_parent[$parent])) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$seen_parent[$parent] = true;
|
||||||
|
if (!self::isKnownCommit($repository, $parent)) {
|
||||||
|
$discover[] = $parent;
|
||||||
|
$insert[] = $parent;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (empty($discover)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
$target = array_pop($insert);
|
||||||
|
list($stdout) = $repository->execxLocalCommand(
|
||||||
|
'log --rev %s --template %s',
|
||||||
|
$target,
|
||||||
|
'{date|rfc822date}');
|
||||||
|
$epoch = strtotime($stdout);
|
||||||
|
|
||||||
|
self::recordCommit($repository, $target, $epoch);
|
||||||
|
|
||||||
|
if (empty($insert)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/* -( Subversion Implementation )------------------------------------------ */
|
||||||
|
|
||||||
|
|
||||||
|
private static function executeSvnDiscover(
|
||||||
|
PhabricatorRepository $repository) {
|
||||||
|
|
||||||
|
$uri = self::executeSvnGetBaseSVNLogURI($repository);
|
||||||
|
|
||||||
|
list($xml) = $repository->execxRemoteCommand(
|
||||||
|
'log --xml --quiet --limit 1 %s@HEAD',
|
||||||
|
$uri);
|
||||||
|
|
||||||
|
$results = self::executeSvnParseLogXML($xml);
|
||||||
|
$commit = head_key($results);
|
||||||
|
$epoch = head($results);
|
||||||
|
|
||||||
|
if (self::isKnownCommit($repository, $commit)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
self::executeSvnDiscoverCommit($repository, $commit, $epoch);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function executeSvnDiscoverCommit(
|
||||||
|
PhabricatorRepository $repository,
|
||||||
|
$commit,
|
||||||
|
$epoch) {
|
||||||
|
|
||||||
|
$uri = self::executeSvnGetBaseSVNLogURI($repository);
|
||||||
|
|
||||||
|
$discover = array(
|
||||||
|
$commit => $epoch,
|
||||||
|
);
|
||||||
|
$upper_bound = $commit;
|
||||||
|
|
||||||
|
$limit = 1;
|
||||||
|
while ($upper_bound > 1 &&
|
||||||
|
!self::isKnownCommit($repository, $upper_bound)) {
|
||||||
|
// Find all the unknown commits on this path. Note that we permit
|
||||||
|
// importing an SVN subdirectory rather than the entire repository, so
|
||||||
|
// commits may be nonsequential.
|
||||||
|
list($err, $xml, $stderr) = $repository->execRemoteCommand(
|
||||||
|
' log --xml --quiet --limit %d %s@%d',
|
||||||
|
$limit,
|
||||||
|
$uri,
|
||||||
|
$upper_bound - 1);
|
||||||
|
if ($err) {
|
||||||
|
if (preg_match('/(path|File) not found/', $stderr)) {
|
||||||
|
// We've gone all the way back through history and this path was not
|
||||||
|
// affected by earlier commits.
|
||||||
|
break;
|
||||||
|
} else {
|
||||||
|
throw new Exception("svn log error #{$err}: {$stderr}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$discover += self::executeSvnParseLogXML($xml);
|
||||||
|
|
||||||
|
$upper_bound = min(array_keys($discover));
|
||||||
|
|
||||||
|
// Discover 2, 4, 8, ... 256 logs at a time. This allows us to initially
|
||||||
|
// import large repositories fairly quickly, while pulling only as much
|
||||||
|
// data as we need in the common case (when we've already imported the
|
||||||
|
// repository and are just grabbing one commit at a time).
|
||||||
|
$limit = min($limit * 2, 256);
|
||||||
|
}
|
||||||
|
|
||||||
|
// NOTE: We do writes only after discovering all the commits so that we're
|
||||||
|
// never left in a state where we've missed commits -- if the discovery
|
||||||
|
// script terminates it can always resume and restore the import to a good
|
||||||
|
// state. This is also why we sort the discovered commits so we can do
|
||||||
|
// writes forward from the smallest one.
|
||||||
|
|
||||||
|
ksort($discover);
|
||||||
|
foreach ($discover as $commit => $epoch) {
|
||||||
|
self::recordCommit($repository, $commit, $epoch);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function executeSvnParseLogXML($xml) {
|
||||||
|
$xml = phutil_utf8ize($xml);
|
||||||
|
|
||||||
|
$result = array();
|
||||||
|
|
||||||
|
$log = new SimpleXMLElement($xml);
|
||||||
|
foreach ($log->logentry as $entry) {
|
||||||
|
$commit = (int)$entry['revision'];
|
||||||
|
$epoch = (int)strtotime((string)$entry->date[0]);
|
||||||
|
$result[$commit] = $epoch;
|
||||||
|
}
|
||||||
|
|
||||||
|
return $result;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static function executeSvnGetBaseSVNLogURI(
|
||||||
|
PhabricatorRepository $repository) {
|
||||||
|
|
||||||
|
$uri = $repository->getDetail('remote-uri');
|
||||||
|
$subpath = $repository->getDetail('svn-subpath');
|
||||||
|
|
||||||
|
return $uri.$subpath;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,12 +6,20 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
phutil_require_module('arcanist', 'repository/parser/mercurial');
|
||||||
|
|
||||||
|
phutil_require_module('phabricator', 'applications/diffusion/data/branch');
|
||||||
|
phutil_require_module('phabricator', 'applications/diffusion/query/branch/git');
|
||||||
phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
|
phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
|
||||||
|
phutil_require_module('phabricator', 'applications/repository/storage/commit');
|
||||||
phutil_require_module('phabricator', 'applications/repository/storage/repository');
|
phutil_require_module('phabricator', 'applications/repository/storage/repository');
|
||||||
phutil_require_module('phabricator', 'infrastructure/daemon/base');
|
phutil_require_module('phabricator', 'infrastructure/daemon/base');
|
||||||
|
phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/event');
|
||||||
|
phutil_require_module('phabricator', 'storage/queryfx');
|
||||||
|
|
||||||
phutil_require_module('phutil', 'error');
|
phutil_require_module('phutil', 'error');
|
||||||
phutil_require_module('phutil', 'filesystem');
|
phutil_require_module('phutil', 'filesystem');
|
||||||
|
phutil_require_module('phutil', 'parser/argument/parser');
|
||||||
phutil_require_module('phutil', 'utils');
|
phutil_require_module('phutil', 'utils');
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -16,10 +16,10 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
final class PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase
|
final class PhabricatorRepositoryPullLocalDaemonTestCase
|
||||||
extends PhabricatorTestCase {
|
extends PhabricatorTestCase {
|
||||||
|
|
||||||
public function testVerifySameGitOrigin() {
|
public function testExecuteGitVerifySameOrigin() {
|
||||||
$cases = array(
|
$cases = array(
|
||||||
array(
|
array(
|
||||||
'ssh://user@domain.com/path.git',
|
'ssh://user@domain.com/path.git',
|
||||||
|
@ -94,7 +94,7 @@ final class PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase
|
||||||
|
|
||||||
$ex = null;
|
$ex = null;
|
||||||
try {
|
try {
|
||||||
PhabricatorRepositoryGitCommitDiscoveryDaemon::verifySameGitOrigin(
|
PhabricatorRepositoryPullLocalDaemon::executeGitverifySameOrigin(
|
||||||
$remote,
|
$remote,
|
||||||
$config,
|
$config,
|
||||||
'(a test case)');
|
'(a test case)');
|
|
@ -6,8 +6,8 @@
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
phutil_require_module('phabricator', 'applications/repository/daemon/commitdiscovery/git');
|
phutil_require_module('phabricator', 'applications/repository/daemon/pulllocal');
|
||||||
phutil_require_module('phabricator', 'infrastructure/testing/testcase');
|
phutil_require_module('phabricator', 'infrastructure/testing/testcase');
|
||||||
|
|
||||||
|
|
||||||
phutil_require_source('PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase.php');
|
phutil_require_source('PhabricatorRepositoryPullLocalDaemonTestCase.php');
|
Loading…
Reference in a new issue