mirror of
https://we.phorge.it/source/phorge.git
synced 2024-11-09 16:32:39 +01:00
Use one daemon to discover commits in all repositories, not one per repository
Summary: See D2418. This merges the commit discovery daemon into the same single daemon, and applies all the same rules to it. There are relatively few implementation changes, but a few things did change: - I simplified/improved Mercurial importing, by finding full branch tip hashes with "--debug branches" and using "parents --template {node}" so we don't need to do separate "--debug id" calls. - Added a new "--not" flag to exclude repositories, since I switched to real arg parsing anyway. - I removed a web UI notification that you need to restart the daemons, this is no longer true. - I added a web UI notification that no pull daemon is running on the machine. NOTE: @makinde, this doesn't change anything from your perspective, but it something breaks this is the likely cause. This implicitly resolves T792, because discovery no longer runs before pulling. Test Plan: - Swapped databases to a fresh install. - Ran "pulllocal" in debug mode. Verified it correctly does nothing (fixed a minor issue with min() on empty array). - Added an SVN repository. Verified it cloned and discovered correctly. - Added a Mercurial repository. Verified it cloned and discovered correctly. - Added a Git repository. Verified it cloned and discovered correctly. - Ran with arguments to verify behaviors: "--not MTEST --not STEST", "P --no-discovery", "P". Reviewers: btrahan, csilvers, Makinde Reviewed By: btrahan CC: aran Maniphest Tasks: T792 Differential Revision: https://secure.phabricator.com/D2430
This commit is contained in:
epriestley
parent
679f778235
commit
d2b01aead0
24 changed files with 637 additions and 748 deletions
|
|
@ -52,13 +52,20 @@ switch (isset($argv[1]) ? $argv[1] : 'help') {
|
|||
$need_launch = phd_load_tracked_repositories();
|
||||
if (!$need_launch) {
|
||||
echo "There are no repositories with tracking enabled.\n";
|
||||
exit(0);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
will_launch($control);
|
||||
|
||||
echo "Launching PullLocal daemon in readonly mode...\n";
|
||||
|
||||
$control->launchDaemon(
|
||||
'PhabricatorRepositoryPullLocalDaemon',
|
||||
array());
|
||||
array(
|
||||
'--no-discovery',
|
||||
));
|
||||
|
||||
echo "Done.\n";
|
||||
break;
|
||||
|
||||
case 'repository-launch-master':
|
||||
|
|
@ -66,55 +73,24 @@ switch (isset($argv[1]) ? $argv[1] : 'help') {
|
|||
if (!$need_launch) {
|
||||
echo "There are no repositories with tracking enabled.\n";
|
||||
exit(1);
|
||||
} else {
|
||||
will_launch($control);
|
||||
|
||||
$control->launchDaemon(
|
||||
'PhabricatorRepositoryPullLocalDaemon',
|
||||
array());
|
||||
|
||||
foreach ($need_launch as $repository) {
|
||||
$name = $repository->getName();
|
||||
$callsign = $repository->getCallsign();
|
||||
$desc = "'{$name}' ({$callsign})";
|
||||
$phid = $repository->getPHID();
|
||||
|
||||
switch ($repository->getVersionControlSystem()) {
|
||||
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
|
||||
echo "Launching discovery daemon on the {$desc} repository...\n";
|
||||
$control->launchDaemon(
|
||||
'PhabricatorRepositoryGitCommitDiscoveryDaemon',
|
||||
array(
|
||||
$phid,
|
||||
));
|
||||
break;
|
||||
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
|
||||
echo "Launching discovery daemon on the {$desc} repository...\n";
|
||||
$control->launchDaemon(
|
||||
'PhabricatorRepositorySvnCommitDiscoveryDaemon',
|
||||
array(
|
||||
$phid,
|
||||
));
|
||||
break;
|
||||
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
|
||||
echo "Launching discovery daemon on the {$desc} repository...\n";
|
||||
$control->launchDaemon(
|
||||
'PhabricatorRepositoryMercurialCommitDiscoveryDaemon',
|
||||
array(
|
||||
$phid,
|
||||
));
|
||||
break;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
echo "Launching CommitTask daemon...\n";
|
||||
$control->launchDaemon(
|
||||
'PhabricatorRepositoryCommitTaskDaemon',
|
||||
array());
|
||||
|
||||
echo "Done.\n";
|
||||
}
|
||||
|
||||
will_launch($control);
|
||||
|
||||
echo "Launching PullLocal daemon in master mode...\n";
|
||||
$control->launchDaemon(
|
||||
'PhabricatorRepositoryPullLocalDaemon',
|
||||
array());
|
||||
|
||||
echo "Launching CommitTask daemon...\n";
|
||||
$control->launchDaemon(
|
||||
'PhabricatorRepositoryCommitTaskDaemon',
|
||||
array());
|
||||
|
||||
echo "NOTE: Make sure you run some taskmaster daemons too, e.g. ".
|
||||
"with 'phd launch 4 taskmaster'.\n";
|
||||
|
||||
echo "Done.\n";
|
||||
break;
|
||||
|
||||
case 'launch':
|
||||
|
|
|
|||
51
scripts/repository/discover.php
Executable file
51
scripts/repository/discover.php
Executable file
|
|
@ -0,0 +1,51 @@
|
|||
#!/usr/bin/env php
|
||||
<?php
|
||||
|
||||
/*
|
||||
* Copyright 2012 Facebook, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
$root = dirname(dirname(dirname(__FILE__)));
|
||||
require_once $root.'/scripts/__init_script__.php';
|
||||
|
||||
$args = new PhutilArgumentParser($argv);
|
||||
$args->setTagline('manually discover working copies');
|
||||
$args->setSynopsis(<<<EOHELP
|
||||
**discover.php** [__options__] __repository-callsign-or-phid ...__
|
||||
Manually discover commits in working copies for the named repositories.
|
||||
EOHELP
|
||||
);
|
||||
$args->parseStandardArguments();
|
||||
$args->parse(
|
||||
array(
|
||||
array(
|
||||
'name' => 'repositories',
|
||||
'wildcard' => true,
|
||||
),
|
||||
));
|
||||
|
||||
$repo_names = $args->getArg('repositories');
|
||||
if (!$repo_names) {
|
||||
echo "Specify one or more repositories to pull, by callsign or PHID.\n";
|
||||
exit(1);
|
||||
}
|
||||
|
||||
$repos = PhabricatorRepository::loadAllByPHIDOrCallsign($repo_names);
|
||||
foreach ($repos as $repo) {
|
||||
$callsign = $repo->getCallsign();
|
||||
echo "Discovering '{$callsign}'...\n";
|
||||
PhabricatorRepositoryPullLocalDaemon::discoverRepository($repo);
|
||||
}
|
||||
echo "Done.\n";
|
||||
|
|
@ -840,7 +840,6 @@ phutil_register_library_map(array(
|
|||
'PhabricatorRepositoryCommit' => 'applications/repository/storage/commit',
|
||||
'PhabricatorRepositoryCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/base',
|
||||
'PhabricatorRepositoryCommitData' => 'applications/repository/storage/commitdata',
|
||||
'PhabricatorRepositoryCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/base',
|
||||
'PhabricatorRepositoryCommitHeraldWorker' => 'applications/repository/worker/herald',
|
||||
'PhabricatorRepositoryCommitMessageDetailParser' => 'applications/repository/parser/base',
|
||||
'PhabricatorRepositoryCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/base',
|
||||
|
|
@ -850,22 +849,18 @@ phutil_register_library_map(array(
|
|||
'PhabricatorRepositoryController' => 'applications/repository/controller/base',
|
||||
'PhabricatorRepositoryCreateController' => 'applications/repository/controller/create',
|
||||
'PhabricatorRepositoryDAO' => 'applications/repository/storage/base',
|
||||
'PhabricatorRepositoryDaemon' => 'applications/repository/daemon/base',
|
||||
'PhabricatorRepositoryDefaultCommitMessageDetailParser' => 'applications/repository/parser/default',
|
||||
'PhabricatorRepositoryDeleteController' => 'applications/repository/controller/delete',
|
||||
'PhabricatorRepositoryEditController' => 'applications/repository/controller/edit',
|
||||
'PhabricatorRepositoryGitCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/git',
|
||||
'PhabricatorRepositoryGitCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/git',
|
||||
'PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase' => 'applications/repository/daemon/commitdiscovery/git/__tests__',
|
||||
'PhabricatorRepositoryGitCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/git',
|
||||
'PhabricatorRepositoryListController' => 'applications/repository/controller/list',
|
||||
'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/mercurial',
|
||||
'PhabricatorRepositoryMercurialCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/mercurial',
|
||||
'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/mercurial',
|
||||
'PhabricatorRepositoryPullLocalDaemon' => 'applications/repository/daemon/pulllocal',
|
||||
'PhabricatorRepositoryPullLocalDaemonTestCase' => 'applications/repository/daemon/pulllocal/__tests__',
|
||||
'PhabricatorRepositoryShortcut' => 'applications/repository/storage/shortcut',
|
||||
'PhabricatorRepositorySvnCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/svn',
|
||||
'PhabricatorRepositorySvnCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/svn',
|
||||
'PhabricatorRepositorySvnCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/svn',
|
||||
'PhabricatorRepositorySymbol' => 'applications/repository/storage/symbol',
|
||||
'PhabricatorRepositoryTestCase' => 'applications/repository/storage/repository/__tests__',
|
||||
|
|
@ -1735,31 +1730,26 @@ phutil_register_library_map(array(
|
|||
'PhabricatorRepositoryCommit' => 'PhabricatorRepositoryDAO',
|
||||
'PhabricatorRepositoryCommitChangeParserWorker' => 'PhabricatorRepositoryCommitParserWorker',
|
||||
'PhabricatorRepositoryCommitData' => 'PhabricatorRepositoryDAO',
|
||||
'PhabricatorRepositoryCommitDiscoveryDaemon' => 'PhabricatorRepositoryDaemon',
|
||||
'PhabricatorRepositoryCommitHeraldWorker' => 'PhabricatorRepositoryCommitParserWorker',
|
||||
'PhabricatorRepositoryCommitMessageParserWorker' => 'PhabricatorRepositoryCommitParserWorker',
|
||||
'PhabricatorRepositoryCommitOwnersWorker' => 'PhabricatorRepositoryCommitParserWorker',
|
||||
'PhabricatorRepositoryCommitParserWorker' => 'PhabricatorWorker',
|
||||
'PhabricatorRepositoryCommitTaskDaemon' => 'PhabricatorRepositoryDaemon',
|
||||
'PhabricatorRepositoryCommitTaskDaemon' => 'PhabricatorDaemon',
|
||||
'PhabricatorRepositoryController' => 'PhabricatorController',
|
||||
'PhabricatorRepositoryCreateController' => 'PhabricatorRepositoryController',
|
||||
'PhabricatorRepositoryDAO' => 'PhabricatorLiskDAO',
|
||||
'PhabricatorRepositoryDaemon' => 'PhabricatorDaemon',
|
||||
'PhabricatorRepositoryDefaultCommitMessageDetailParser' => 'PhabricatorRepositoryCommitMessageDetailParser',
|
||||
'PhabricatorRepositoryDeleteController' => 'PhabricatorRepositoryController',
|
||||
'PhabricatorRepositoryEditController' => 'PhabricatorRepositoryController',
|
||||
'PhabricatorRepositoryGitCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
|
||||
'PhabricatorRepositoryGitCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
|
||||
'PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase' => 'PhabricatorTestCase',
|
||||
'PhabricatorRepositoryGitCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
|
||||
'PhabricatorRepositoryListController' => 'PhabricatorRepositoryController',
|
||||
'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
|
||||
'PhabricatorRepositoryMercurialCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
|
||||
'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
|
||||
'PhabricatorRepositoryPullLocalDaemon' => 'PhabricatorDaemon',
|
||||
'PhabricatorRepositoryPullLocalDaemonTestCase' => 'PhabricatorTestCase',
|
||||
'PhabricatorRepositoryShortcut' => 'PhabricatorRepositoryDAO',
|
||||
'PhabricatorRepositorySvnCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
|
||||
'PhabricatorRepositorySvnCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
|
||||
'PhabricatorRepositorySvnCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
|
||||
'PhabricatorRepositorySymbol' => 'PhabricatorRepositoryDAO',
|
||||
'PhabricatorRepositoryTestCase' => 'PhabricatorTestCase',
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
<?php
|
||||
|
||||
/*
|
||||
* Copyright 2011 Facebook, Inc.
|
||||
* Copyright 2012 Facebook, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -33,7 +33,11 @@ final class DiffusionSvnHistoryQuery extends DiffusionHistoryQuery {
|
|||
PhabricatorRepository::TABLE_PATH,
|
||||
array(md5('/'.trim($path, '/'))));
|
||||
$paths = ipull($paths, 'id', 'path');
|
||||
$path_id = $paths['/'.trim($path, '/')];
|
||||
$path_id = idx($paths, '/'.trim($path, '/'));
|
||||
|
||||
if (!$path_id) {
|
||||
return array();
|
||||
}
|
||||
|
||||
$filter_query = '';
|
||||
if ($this->needDirectChanges) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
<?php
|
||||
|
||||
/*
|
||||
* Copyright 2011 Facebook, Inc.
|
||||
* Copyright 2012 Facebook, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -32,8 +32,43 @@ abstract class PhabricatorRepositoryController extends PhabricatorController {
|
|||
$page->setGlyph("rX");
|
||||
$page->appendChild($view);
|
||||
|
||||
|
||||
$response = new AphrontWebpageResponse();
|
||||
return $response->setContent($page->render());
|
||||
}
|
||||
|
||||
private function isPullDaemonRunningOnThisMachine() {
|
||||
|
||||
// This is sort of hacky, but should probably work.
|
||||
|
||||
list($stdout) = execx('ps auxwww');
|
||||
return preg_match('/PhabricatorRepositoryPullLocalDaemon/', $stdout);
|
||||
}
|
||||
|
||||
protected function renderDaemonNotice() {
|
||||
$daemon_running = $this->isPullDaemonRunningOnThisMachine();
|
||||
if ($daemon_running) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$documentation = phutil_render_tag(
|
||||
'a',
|
||||
array(
|
||||
'href' => PhabricatorEnv::getDoclink(
|
||||
'article/Diffusion_User_Guide.html'),
|
||||
),
|
||||
'Diffusion User Guide');
|
||||
|
||||
$view = new AphrontErrorView();
|
||||
$view->setSeverity(AphrontErrorView::SEVERITY_WARNING);
|
||||
$view->setTitle('Repository Daemon Not Running');
|
||||
$view->appendChild(
|
||||
"<p>The repository daemon is not running on this machine. Without this ".
|
||||
"daemon, Phabricator will not be able to import or update repositories. ".
|
||||
"For instructions on starting the daemon, see ".
|
||||
"<strong>{$documentation}</strong>.</p>");
|
||||
|
||||
return $view;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,11 @@
|
|||
|
||||
phutil_require_module('phabricator', 'aphront/response/webpage');
|
||||
phutil_require_module('phabricator', 'applications/base/controller/base');
|
||||
phutil_require_module('phabricator', 'infrastructure/env');
|
||||
phutil_require_module('phabricator', 'view/form/error');
|
||||
|
||||
phutil_require_module('phutil', 'future/exec');
|
||||
phutil_require_module('phutil', 'markup');
|
||||
phutil_require_module('phutil', 'utils');
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -63,6 +63,8 @@ final class PhabricatorRepositoryEditController
|
|||
phutil_escape_html($name)));
|
||||
}
|
||||
|
||||
$nav->appendChild($this->renderDaemonNotice());
|
||||
|
||||
$this->sideNav = $nav;
|
||||
|
||||
switch ($this->view) {
|
||||
|
|
@ -345,9 +347,7 @@ final class PhabricatorRepositoryEditController
|
|||
$error_view = new AphrontErrorView();
|
||||
$error_view->setSeverity(AphrontErrorView::SEVERITY_NOTICE);
|
||||
$error_view->setTitle('Changes Saved');
|
||||
$error_view->appendChild(
|
||||
'Tracking changes were saved. You may need to restart the daemon '.
|
||||
'before changes will take effect.');
|
||||
$error_view->appendChild('Tracking changes were saved.');
|
||||
} else if (!$repository->isTracked()) {
|
||||
$error_view = new AphrontErrorView();
|
||||
$error_view->setSeverity(AphrontErrorView::SEVERITY_WARNING);
|
||||
|
|
|
|||
|
|
@ -157,6 +157,7 @@ final class PhabricatorRepositoryListController
|
|||
|
||||
return $this->buildStandardPageResponse(
|
||||
array(
|
||||
$this->renderDaemonNotice(),
|
||||
$panel,
|
||||
$project_panel,
|
||||
),
|
||||
|
|
|
|||
|
|
@ -1,38 +0,0 @@
|
|||
<?php
|
||||
|
||||
/*
|
||||
* Copyright 2011 Facebook, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
abstract class PhabricatorRepositoryDaemon extends PhabricatorDaemon {
|
||||
|
||||
protected function loadRepository() {
|
||||
$argv = $this->getArgv();
|
||||
if (count($argv) !== 1) {
|
||||
throw new Exception("No repository PHID provided!");
|
||||
}
|
||||
|
||||
$repository = id(new PhabricatorRepository())->loadOneWhere(
|
||||
'phid = %s',
|
||||
$argv[0]);
|
||||
|
||||
if (!$repository) {
|
||||
throw new Exception("No such repository exists!");
|
||||
}
|
||||
|
||||
return $repository;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
<?php
|
||||
/**
|
||||
* This file is automatically generated. Lint this module to rebuild it.
|
||||
* @generated
|
||||
*/
|
||||
|
||||
|
||||
|
||||
phutil_require_module('phabricator', 'applications/repository/storage/repository');
|
||||
phutil_require_module('phabricator', 'infrastructure/daemon/base');
|
||||
|
||||
phutil_require_module('phutil', 'utils');
|
||||
|
||||
|
||||
phutil_require_source('PhabricatorRepositoryDaemon.php');
|
||||
|
|
@ -1,113 +0,0 @@
|
|||
<?php
|
||||
|
||||
/*
|
||||
* Copyright 2012 Facebook, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
abstract class PhabricatorRepositoryCommitDiscoveryDaemon
|
||||
extends PhabricatorRepositoryDaemon {
|
||||
|
||||
private $repository;
|
||||
private $commitCache = array();
|
||||
|
||||
final protected function getRepository() {
|
||||
return $this->repository;
|
||||
}
|
||||
|
||||
final public function run() {
|
||||
while (true) {
|
||||
// Reload the repository every time to pick up changes from the web
|
||||
// console.
|
||||
$this->repository = $this->loadRepository();
|
||||
$this->discoverCommits();
|
||||
|
||||
$sleep = max(2, $this->getRepository()->getDetail('pull-frequency'));
|
||||
$this->sleep($sleep);
|
||||
}
|
||||
}
|
||||
|
||||
final public function runOnce() {
|
||||
$this->repository = $this->loadRepository();
|
||||
$this->discoverCommits();
|
||||
}
|
||||
|
||||
protected function isKnownCommit($target) {
|
||||
if (isset($this->commitCache[$target])) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
|
||||
'repositoryID = %s AND commitIdentifier = %s',
|
||||
$this->getRepository()->getID(),
|
||||
$target);
|
||||
|
||||
if (!$commit) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->commitCache[$target] = true;
|
||||
while (count($this->commitCache) > 64) {
|
||||
array_shift($this->commitCache);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
protected function recordCommit($commit_identifier, $epoch) {
|
||||
$repository = $this->getRepository();
|
||||
|
||||
$commit = new PhabricatorRepositoryCommit();
|
||||
$commit->setRepositoryID($repository->getID());
|
||||
$commit->setCommitIdentifier($commit_identifier);
|
||||
$commit->setEpoch($epoch);
|
||||
|
||||
try {
|
||||
$commit->save();
|
||||
$event = new PhabricatorTimelineEvent(
|
||||
'cmit',
|
||||
array(
|
||||
'id' => $commit->getID(),
|
||||
));
|
||||
$event->recordEvent();
|
||||
|
||||
queryfx(
|
||||
$repository->establishConnection('w'),
|
||||
'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
|
||||
VALUES (%d, 1, %d, %d)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
size = size + 1,
|
||||
lastCommitID =
|
||||
IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID),
|
||||
epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)',
|
||||
PhabricatorRepository::TABLE_SUMMARY,
|
||||
$repository->getID(),
|
||||
$commit->getID(),
|
||||
$epoch);
|
||||
|
||||
$this->commitCache[$commit_identifier] = true;
|
||||
} catch (AphrontQueryDuplicateKeyException $ex) {
|
||||
// Ignore. This can happen because we discover the same new commit
|
||||
// more than once when looking at history, or because of races or
|
||||
// data inconsistency or cosmic radiation; in any case, we're still
|
||||
// in a good state if we ignore the failure.
|
||||
$this->commitCache[$commit_identifier] = true;
|
||||
}
|
||||
|
||||
$this->stillWorking();
|
||||
}
|
||||
|
||||
abstract protected function discoverCommits();
|
||||
|
||||
}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
<?php
|
||||
/**
|
||||
* This file is automatically generated. Lint this module to rebuild it.
|
||||
* @generated
|
||||
*/
|
||||
|
||||
|
||||
|
||||
phutil_require_module('phabricator', 'applications/repository/daemon/base');
|
||||
phutil_require_module('phabricator', 'applications/repository/storage/commit');
|
||||
phutil_require_module('phabricator', 'applications/repository/storage/repository');
|
||||
phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/event');
|
||||
phutil_require_module('phabricator', 'storage/queryfx');
|
||||
|
||||
phutil_require_module('phutil', 'utils');
|
||||
|
||||
|
||||
phutil_require_source('PhabricatorRepositoryCommitDiscoveryDaemon.php');
|
||||
|
|
@ -1,162 +0,0 @@
|
|||
<?php
|
||||
|
||||
/*
|
||||
* Copyright 2012 Facebook, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
final class PhabricatorRepositoryGitCommitDiscoveryDaemon
|
||||
extends PhabricatorRepositoryCommitDiscoveryDaemon {
|
||||
|
||||
protected function discoverCommits() {
|
||||
// NOTE: PhabricatorRepositoryGitFetchDaemon does the actual pulls, this
|
||||
// just parses HEAD.
|
||||
|
||||
$repository = $this->getRepository();
|
||||
|
||||
$vcs = $repository->getVersionControlSystem();
|
||||
if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_GIT) {
|
||||
throw new Exception("Repository is not a git repository.");
|
||||
}
|
||||
|
||||
list($remotes) = $repository->execxLocalCommand(
|
||||
'remote show -n origin');
|
||||
|
||||
$matches = null;
|
||||
if (!preg_match('/^\s*Fetch URL:\s*(.*?)\s*$/m', $remotes, $matches)) {
|
||||
throw new Exception(
|
||||
"Expected 'Fetch URL' in 'git remote show -n origin'.");
|
||||
}
|
||||
|
||||
self::verifySameGitOrigin(
|
||||
$matches[1],
|
||||
$repository->getRemoteURI(),
|
||||
$repository->getLocalPath());
|
||||
|
||||
list($stdout) = $repository->execxLocalCommand(
|
||||
'branch -r --verbose --no-abbrev');
|
||||
|
||||
$branches = DiffusionGitBranchQuery::parseGitRemoteBranchOutput(
|
||||
$stdout,
|
||||
$only_this_remote = DiffusionBranchInformation::DEFAULT_GIT_REMOTE);
|
||||
|
||||
$got_something = false;
|
||||
$tracked_something = false;
|
||||
foreach ($branches as $name => $commit) {
|
||||
if (!$repository->shouldTrackBranch($name)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$tracked_something = true;
|
||||
|
||||
if ($this->isKnownCommit($commit)) {
|
||||
continue;
|
||||
} else {
|
||||
$this->discoverCommit($commit);
|
||||
$got_something = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!$tracked_something) {
|
||||
$repo_name = $repository->getName();
|
||||
$repo_callsign = $repository->getCallsign();
|
||||
throw new Exception(
|
||||
"Repository r{$repo_callsign} '{$repo_name}' has no tracked branches! ".
|
||||
"Verify that your branch filtering settings are correct.");
|
||||
}
|
||||
|
||||
return $got_something;
|
||||
}
|
||||
|
||||
private function discoverCommit($commit) {
|
||||
$discover = array();
|
||||
$insert = array();
|
||||
|
||||
$repository = $this->getRepository();
|
||||
|
||||
$discover[] = $commit;
|
||||
$insert[] = $commit;
|
||||
|
||||
$seen_parent = array();
|
||||
|
||||
while (true) {
|
||||
$target = array_pop($discover);
|
||||
list($parents) = $repository->execxLocalCommand(
|
||||
'log -n1 --pretty="%%P" %s',
|
||||
$target);
|
||||
$parents = array_filter(explode(' ', trim($parents)));
|
||||
foreach ($parents as $parent) {
|
||||
if (isset($seen_parent[$parent])) {
|
||||
// We end up in a loop here somehow when we parse Arcanist if we
|
||||
// don't do this. TODO: Figure out why and draw a pretty diagram
|
||||
// since it's not evident how parsing a DAG with this causes the
|
||||
// loop to stop terminating.
|
||||
continue;
|
||||
}
|
||||
$seen_parent[$parent] = true;
|
||||
if (!$this->isKnownCommit($parent)) {
|
||||
$discover[] = $parent;
|
||||
$insert[] = $parent;
|
||||
}
|
||||
}
|
||||
if (empty($discover)) {
|
||||
break;
|
||||
}
|
||||
$this->stillWorking();
|
||||
}
|
||||
|
||||
while (true) {
|
||||
$target = array_pop($insert);
|
||||
list($epoch) = $repository->execxLocalCommand(
|
||||
'log -n1 --pretty="%%ct" %s',
|
||||
$target);
|
||||
$epoch = trim($epoch);
|
||||
|
||||
$this->recordCommit($target, $epoch);
|
||||
|
||||
if (empty($insert)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static function verifySameGitOrigin($remote, $expect, $where) {
|
||||
$remote_uri = PhabricatorRepository::newPhutilURIFromGitURI($remote);
|
||||
$expect_uri = PhabricatorRepository::newPhutilURIFromGitURI($expect);
|
||||
|
||||
$remote_path = $remote_uri->getPath();
|
||||
$expect_path = $expect_uri->getPath();
|
||||
|
||||
$remote_match = self::normalizeGitPath($remote_path);
|
||||
$expect_match = self::normalizeGitPath($expect_path);
|
||||
|
||||
if ($remote_match != $expect_match) {
|
||||
throw new Exception(
|
||||
"Working copy at '{$where}' has a mismatched origin URL. It has ".
|
||||
"origin URL '{$remote}' (with remote path '{$remote_path}'), but the ".
|
||||
"configured URL '{$expect}' (with remote path '{$expect_path}') is ".
|
||||
"expected. Refusing to proceed because this may indicate that the ".
|
||||
"working copy is actually some other repository.");
|
||||
}
|
||||
}
|
||||
|
||||
private static function normalizeGitPath($path) {
|
||||
// Strip away trailing "/" and ".git", so similar paths correctly match.
|
||||
|
||||
$path = rtrim($path, '/');
|
||||
$path = preg_replace('/\.git$/', '', $path);
|
||||
return $path;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
<?php
|
||||
/**
|
||||
* This file is automatically generated. Lint this module to rebuild it.
|
||||
* @generated
|
||||
*/
|
||||
|
||||
|
||||
|
||||
phutil_require_module('phabricator', 'applications/diffusion/data/branch');
|
||||
phutil_require_module('phabricator', 'applications/diffusion/query/branch/git');
|
||||
phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
|
||||
phutil_require_module('phabricator', 'applications/repository/daemon/commitdiscovery/base');
|
||||
phutil_require_module('phabricator', 'applications/repository/storage/repository');
|
||||
|
||||
|
||||
phutil_require_source('PhabricatorRepositoryGitCommitDiscoveryDaemon.php');
|
||||
|
|
@ -1,137 +0,0 @@
|
|||
<?php
|
||||
|
||||
/*
|
||||
* Copyright 2012 Facebook, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
final class PhabricatorRepositoryMercurialCommitDiscoveryDaemon
|
||||
extends PhabricatorRepositoryCommitDiscoveryDaemon {
|
||||
|
||||
protected function discoverCommits() {
|
||||
$repository = $this->getRepository();
|
||||
|
||||
$vcs = $repository->getVersionControlSystem();
|
||||
if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL) {
|
||||
throw new Exception("Repository is not a Mercurial repository.");
|
||||
}
|
||||
|
||||
$repository_phid = $repository->getPHID();
|
||||
|
||||
list($stdout) = $repository->execxLocalCommand('branches');
|
||||
|
||||
$branches = ArcanistMercurialParser::parseMercurialBranches($stdout);
|
||||
$got_something = false;
|
||||
foreach ($branches as $name => $branch) {
|
||||
$commit = $branch['rev'];
|
||||
$commit = $this->getFullHash($commit);
|
||||
if ($this->isKnownCommit($commit)) {
|
||||
continue;
|
||||
} else {
|
||||
$this->discoverCommit($commit);
|
||||
$got_something = true;
|
||||
}
|
||||
}
|
||||
|
||||
return $got_something;
|
||||
}
|
||||
|
||||
private function getFullHash($commit) {
|
||||
|
||||
// NOTE: Mercurial shortens hashes to 12 characters by default. This
|
||||
// implies collisions with as few as a few million commits. The
|
||||
// documentation sensibly advises "Do not use short-form IDs for
|
||||
// long-lived representations". It then continues "You can use the
|
||||
// --debug option to display the full changeset ID". What?! Yes, this
|
||||
// is in fact the only way to turn on full hashes, and the hg source
|
||||
// code is littered with "hexfn = ui.debugflag and hex or short" and
|
||||
// similar. There is no more-selective flag or config option.
|
||||
//
|
||||
// Unfortunately, "hg --debug" turns on tons of other extra output,
|
||||
// including full commit messages in "hg log" and "hg parents" (which
|
||||
// ignore --style); this renders them unparseable. So we have to use
|
||||
// "hg id" to convert short hashes into full hashes. See:
|
||||
//
|
||||
// <http://mercurial.selenic.com/wiki/ChangeSetID>
|
||||
//
|
||||
// Of course, this means that if there are collisions we will break here
|
||||
// (the short commit identifier won't be unambiguous) but maybe Mercurial
|
||||
// will have a --full-hashes flag or something by then and we can fix it
|
||||
// properly. Until we run into that, this allows us to store data in the
|
||||
// right format so when we eventually encounter this we won't have to
|
||||
// reparse every Mercurial repository.
|
||||
|
||||
$repository = $this->getRepository();
|
||||
list($stdout) = $repository->execxLocalCommand(
|
||||
'id --debug -i --rev %s',
|
||||
$commit);
|
||||
return trim($stdout);
|
||||
}
|
||||
|
||||
private function discoverCommit($commit) {
|
||||
$discover = array();
|
||||
$insert = array();
|
||||
|
||||
$repository = $this->getRepository();
|
||||
|
||||
$discover[] = $commit;
|
||||
$insert[] = $commit;
|
||||
|
||||
$seen_parent = array();
|
||||
|
||||
// For all the new commits at the branch heads, walk backward until we find
|
||||
// only commits we've aleady seen.
|
||||
while (true) {
|
||||
$target = array_pop($discover);
|
||||
list($stdout) = $repository->execxLocalCommand(
|
||||
'parents --style default --rev %s',
|
||||
$target);
|
||||
$parents = ArcanistMercurialParser::parseMercurialLog($stdout);
|
||||
if ($parents) {
|
||||
foreach ($parents as $parent) {
|
||||
$parent_commit = $parent['rev'];
|
||||
$parent_commit = $this->getFullHash($parent_commit);
|
||||
if (isset($seen_parent[$parent_commit])) {
|
||||
continue;
|
||||
}
|
||||
$seen_parent[$parent_commit] = true;
|
||||
if (!$this->isKnownCommit($parent_commit)) {
|
||||
$discover[] = $parent_commit;
|
||||
$insert[] = $parent_commit;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (empty($discover)) {
|
||||
break;
|
||||
}
|
||||
$this->stillWorking();
|
||||
}
|
||||
|
||||
while (true) {
|
||||
$target = array_pop($insert);
|
||||
list($stdout) = $repository->execxLocalCommand(
|
||||
'log --rev %s --template %s',
|
||||
$target,
|
||||
'{date|rfc822date}');
|
||||
$epoch = strtotime($stdout);
|
||||
|
||||
$this->recordCommit($target, $epoch);
|
||||
|
||||
if (empty($insert)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
<?php
|
||||
/**
|
||||
* This file is automatically generated. Lint this module to rebuild it.
|
||||
* @generated
|
||||
*/
|
||||
|
||||
|
||||
|
||||
phutil_require_module('arcanist', 'repository/parser/mercurial');
|
||||
|
||||
phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
|
||||
phutil_require_module('phabricator', 'applications/repository/daemon/commitdiscovery/base');
|
||||
|
||||
|
||||
phutil_require_source('PhabricatorRepositoryMercurialCommitDiscoveryDaemon.php');
|
||||
|
|
@ -1,123 +0,0 @@
|
|||
<?php
|
||||
|
||||
/*
|
||||
* Copyright 2012 Facebook, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
final class PhabricatorRepositorySvnCommitDiscoveryDaemon
|
||||
extends PhabricatorRepositoryCommitDiscoveryDaemon {
|
||||
|
||||
protected function discoverCommits() {
|
||||
$repository = $this->getRepository();
|
||||
|
||||
$vcs = $repository->getVersionControlSystem();
|
||||
if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_SVN) {
|
||||
throw new Exception("Repository is not a svn repository.");
|
||||
}
|
||||
|
||||
$uri = $this->getBaseSVNLogURI();
|
||||
list($xml) = $repository->execxRemoteCommand(
|
||||
'log --xml --quiet --limit 1 %s@HEAD',
|
||||
$uri);
|
||||
|
||||
$results = $this->parseSVNLogXML($xml);
|
||||
$commit = head_key($results);
|
||||
$epoch = head($results);
|
||||
|
||||
if ($this->isKnownCommit($commit)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->discoverCommit($commit, $epoch);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function discoverCommit($commit, $epoch) {
|
||||
$uri = $this->getBaseSVNLogURI();
|
||||
$repository = $this->getRepository();
|
||||
|
||||
$discover = array(
|
||||
$commit => $epoch,
|
||||
);
|
||||
$upper_bound = $commit;
|
||||
|
||||
$limit = 1;
|
||||
while ($upper_bound > 1 && !$this->isKnownCommit($upper_bound)) {
|
||||
// Find all the unknown commits on this path. Note that we permit
|
||||
// importing an SVN subdirectory rather than the entire repository, so
|
||||
// commits may be nonsequential.
|
||||
list($err, $xml, $stderr) = $repository->execRemoteCommand(
|
||||
' log --xml --quiet --limit %d %s@%d',
|
||||
$limit,
|
||||
$uri,
|
||||
$upper_bound - 1);
|
||||
if ($err) {
|
||||
if (preg_match('/(path|File) not found/', $stderr)) {
|
||||
// We've gone all the way back through history and this path was not
|
||||
// affected by earlier commits.
|
||||
break;
|
||||
} else {
|
||||
throw new Exception("svn log error #{$err}: {$stderr}");
|
||||
}
|
||||
}
|
||||
$discover += $this->parseSVNLogXML($xml);
|
||||
|
||||
$upper_bound = min(array_keys($discover));
|
||||
|
||||
// Discover 2, 4, 8, ... 256 logs at a time. This allows us to initially
|
||||
// import large repositories fairly quickly, while pulling only as much
|
||||
// data as we need in the common case (when we've already imported the
|
||||
// repository and are just grabbing one commit at a time).
|
||||
$limit = min($limit * 2, 256);
|
||||
}
|
||||
|
||||
// NOTE: We do writes only after discovering all the commits so that we're
|
||||
// never left in a state where we've missed commits -- if the discovery
|
||||
// script terminates it can always resume and restore the import to a good
|
||||
// state. This is also why we sort the discovered commits so we can do
|
||||
// writes forward from the smallest one.
|
||||
|
||||
ksort($discover);
|
||||
foreach ($discover as $commit => $epoch) {
|
||||
$this->recordCommit($commit, $epoch);
|
||||
}
|
||||
}
|
||||
|
||||
private function parseSVNLogXML($xml) {
|
||||
$xml = phutil_utf8ize($xml);
|
||||
|
||||
$result = array();
|
||||
|
||||
$log = new SimpleXMLElement($xml);
|
||||
foreach ($log->logentry as $entry) {
|
||||
$commit = (int)$entry['revision'];
|
||||
$epoch = (int)strtotime((string)$entry->date[0]);
|
||||
$result[$commit] = $epoch;
|
||||
}
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
|
||||
private function getBaseSVNLogURI() {
|
||||
$repository = $this->getRepository();
|
||||
|
||||
$uri = $repository->getDetail('remote-uri');
|
||||
$subpath = $repository->getDetail('svn-subpath');
|
||||
|
||||
return $uri.$subpath;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
<?php
|
||||
/**
|
||||
* This file is automatically generated. Lint this module to rebuild it.
|
||||
* @generated
|
||||
*/
|
||||
|
||||
|
||||
|
||||
phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
|
||||
phutil_require_module('phabricator', 'applications/repository/daemon/commitdiscovery/base');
|
||||
|
||||
phutil_require_module('phutil', 'utils');
|
||||
|
||||
|
||||
phutil_require_source('PhabricatorRepositorySvnCommitDiscoveryDaemon.php');
|
||||
|
|
@ -17,7 +17,7 @@
|
|||
*/
|
||||
|
||||
final class PhabricatorRepositoryCommitTaskDaemon
|
||||
extends PhabricatorRepositoryDaemon {
|
||||
extends PhabricatorDaemon {
|
||||
|
||||
final public function run() {
|
||||
do {
|
||||
|
|
|
|||
|
|
@ -7,9 +7,9 @@
|
|||
|
||||
|
||||
phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
|
||||
phutil_require_module('phabricator', 'applications/repository/daemon/base');
|
||||
phutil_require_module('phabricator', 'applications/repository/storage/commit');
|
||||
phutil_require_module('phabricator', 'applications/repository/storage/repository');
|
||||
phutil_require_module('phabricator', 'infrastructure/daemon/base');
|
||||
phutil_require_module('phabricator', 'infrastructure/daemon/timeline/cursor/iterator');
|
||||
phutil_require_module('phabricator', 'infrastructure/daemon/workers/storage/task');
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,12 @@
|
|||
* responsible for only some repositories, you can launch it with a list of
|
||||
* PHIDs or callsigns:
|
||||
*
|
||||
* ./phd launch repositorypulllocal X Q Z
|
||||
* ./phd launch repositorypulllocal -- X Q Z
|
||||
*
|
||||
* You can also launch a daemon which is responsible for all //but// one or
|
||||
* more repositories:
|
||||
*
|
||||
* ./phd launch repositorypulllocal -- --not A --not B
|
||||
*
|
||||
* If you have a very large number of repositories and some aren't being pulled
|
||||
* as frequently as you'd like, you can either change the pull frequency of
|
||||
|
|
@ -40,6 +45,8 @@
|
|||
final class PhabricatorRepositoryPullLocalDaemon
|
||||
extends PhabricatorDaemon {
|
||||
|
||||
private static $commitCache = array();
|
||||
|
||||
|
||||
/* -( Pulling Repositories )----------------------------------------------- */
|
||||
|
||||
|
|
@ -48,14 +55,45 @@ final class PhabricatorRepositoryPullLocalDaemon
|
|||
* @task pull
|
||||
*/
|
||||
public function run() {
|
||||
$argv = $this->getArgv();
|
||||
array_unshift($argv, __CLASS__);
|
||||
$args = new PhutilArgumentParser($argv);
|
||||
$args->parse(
|
||||
array(
|
||||
array(
|
||||
'name' => 'no-discovery',
|
||||
'help' => 'Pull only, without discovering commits.',
|
||||
),
|
||||
array(
|
||||
'name' => 'not',
|
||||
'param' => 'repository',
|
||||
'repeat' => true,
|
||||
'help' => 'Do not pull __repository__.',
|
||||
),
|
||||
array(
|
||||
'name' => 'repositories',
|
||||
'wildcard' => true,
|
||||
'help' => 'Pull specific __repositories__ instead of all.',
|
||||
),
|
||||
));
|
||||
|
||||
$no_discovery = $args->getArg('no-discovery');
|
||||
$repo_names = $args->getArg('repositories', array());
|
||||
$exclude_names = $args->getArg('not', array());
|
||||
|
||||
// Each repository has an individual pull frequency; after we pull it,
|
||||
// wait that long to pull it again. When we start up, try to pull everything
|
||||
// serially.
|
||||
$retry_after = array();
|
||||
|
||||
$min_sleep = 15;
|
||||
|
||||
while (true) {
|
||||
$repositories = $this->loadRepositories();
|
||||
$repositories = $this->loadRepositories($repo_names);
|
||||
if ($exclude_names) {
|
||||
$exclude = $this->loadRepositories($exclude_names);
|
||||
$repositories = array_diff_key($repositories, $exclude);
|
||||
}
|
||||
|
||||
// Shuffle the repositories, then re-key the array since shuffle()
|
||||
// discards keys. This is mostly for startup, we'll use soft priorities
|
||||
|
|
@ -79,22 +117,41 @@ final class PhabricatorRepositoryPullLocalDaemon
|
|||
|
||||
foreach ($repositories as $id => $repository) {
|
||||
$after = idx($retry_after, $id, 0);
|
||||
if ($after >= time()) {
|
||||
if ($after > time()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$tracked = $repository->isTracked();
|
||||
if (!$tracked) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
self::pullRepository($repository);
|
||||
$sleep_for = $repository->getDetail('pull-frequency', 15);
|
||||
|
||||
if (!$no_discovery) {
|
||||
// TODO: It would be nice to discover only if we pulled something,
|
||||
// but this isn't totally trivial.
|
||||
self::discoverRepository($repository);
|
||||
}
|
||||
|
||||
$sleep_for = $repository->getDetail('pull-frequency', $min_sleep);
|
||||
$retry_after[$id] = time() + $sleep_for;
|
||||
} catch (Exception $ex) {
|
||||
$retry_after[$id] = time() + 15;
|
||||
$retry_after[$id] = time() + $min_sleep;
|
||||
phlog($ex);
|
||||
}
|
||||
|
||||
$this->stillWorking();
|
||||
}
|
||||
|
||||
$sleep_until = max(min($retry_after), time() + 15);
|
||||
sleep($sleep_until - time());
|
||||
if ($retry_after) {
|
||||
$sleep_until = max(min($retry_after), time() + $min_sleep);
|
||||
} else {
|
||||
$sleep_until = time() + $min_sleep;
|
||||
}
|
||||
|
||||
$this->sleep($sleep_until - time());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -102,12 +159,11 @@ final class PhabricatorRepositoryPullLocalDaemon
|
|||
/**
|
||||
* @task pull
|
||||
*/
|
||||
protected function loadRepositories() {
|
||||
$argv = $this->getArgv();
|
||||
if (!count($argv)) {
|
||||
protected function loadRepositories(array $names) {
|
||||
if (!count($names)) {
|
||||
return id(new PhabricatorRepository())->loadAll();
|
||||
} else {
|
||||
return PhabricatorRepository::loadAllByPHIDOrCallsign($argv);
|
||||
return PhabricatorRepository::loadAllByPHIDOrCallsign($names);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -116,11 +172,6 @@ final class PhabricatorRepositoryPullLocalDaemon
|
|||
* @task pull
|
||||
*/
|
||||
public static function pullRepository(PhabricatorRepository $repository) {
|
||||
$tracked = $repository->isTracked();
|
||||
if (!$tracked) {
|
||||
return;
|
||||
}
|
||||
|
||||
$vcs = $repository->getVersionControlSystem();
|
||||
|
||||
$is_svn = ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_SVN);
|
||||
|
|
@ -153,19 +204,126 @@ final class PhabricatorRepositoryPullLocalDaemon
|
|||
}
|
||||
|
||||
if ($is_git) {
|
||||
self::executeGitCreate($repository, $local_path);
|
||||
return self::executeGitCreate($repository, $local_path);
|
||||
} else if ($is_hg) {
|
||||
self::executeHgCreate($repository, $local_path);
|
||||
return self::executeHgCreate($repository, $local_path);
|
||||
}
|
||||
} else {
|
||||
if ($is_git) {
|
||||
self::executeGitUpdate($repository, $local_path);
|
||||
return self::executeGitUpdate($repository, $local_path);
|
||||
} else if ($is_hg) {
|
||||
self::executeHgUpdate($repository, $local_path);
|
||||
return self::executeHgUpdate($repository, $local_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static function discoverRepository(PhabricatorRepository $repository) {
|
||||
$vcs = $repository->getVersionControlSystem();
|
||||
switch ($vcs) {
|
||||
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
|
||||
return self::executeGitDiscover($repository);
|
||||
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
|
||||
return self::executeSvnDiscover($repository);
|
||||
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
|
||||
return self::executeHgDiscover($repository);
|
||||
default:
|
||||
throw new Exception("Unknown VCS '{$vcs}'!");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static function isKnownCommit(
|
||||
PhabricatorRepository $repository,
|
||||
$target) {
|
||||
|
||||
if (self::getCache($repository, $target)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
|
||||
'repositoryID = %s AND commitIdentifier = %s',
|
||||
$repository->getID(),
|
||||
$target);
|
||||
|
||||
if (!$commit) {
|
||||
return false;
|
||||
}
|
||||
|
||||
self::setCache($repository, $target);
|
||||
while (count(self::$commitCache) > 2048) {
|
||||
array_shift(self::$commitCache);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static function recordCommit(
|
||||
PhabricatorRepository $repository,
|
||||
$commit_identifier,
|
||||
$epoch) {
|
||||
|
||||
$commit = new PhabricatorRepositoryCommit();
|
||||
$commit->setRepositoryID($repository->getID());
|
||||
$commit->setCommitIdentifier($commit_identifier);
|
||||
$commit->setEpoch($epoch);
|
||||
|
||||
try {
|
||||
$commit->save();
|
||||
$event = new PhabricatorTimelineEvent(
|
||||
'cmit',
|
||||
array(
|
||||
'id' => $commit->getID(),
|
||||
));
|
||||
$event->recordEvent();
|
||||
|
||||
queryfx(
|
||||
$repository->establishConnection('w'),
|
||||
'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
|
||||
VALUES (%d, 1, %d, %d)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
size = size + 1,
|
||||
lastCommitID =
|
||||
IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID),
|
||||
epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)',
|
||||
PhabricatorRepository::TABLE_SUMMARY,
|
||||
$repository->getID(),
|
||||
$commit->getID(),
|
||||
$epoch);
|
||||
|
||||
self::setCache($repository, $commit_identifier);
|
||||
} catch (AphrontQueryDuplicateKeyException $ex) {
|
||||
// Ignore. This can happen because we discover the same new commit
|
||||
// more than once when looking at history, or because of races or
|
||||
// data inconsistency or cosmic radiation; in any case, we're still
|
||||
// in a good state if we ignore the failure.
|
||||
self::setCache($repository, $commit_identifier);
|
||||
}
|
||||
}
|
||||
|
||||
private static function setCache(
|
||||
PhabricatorRepository $repository,
|
||||
$commit_identifier) {
|
||||
|
||||
$key = self::getCacheKey($repository, $commit_identifier);
|
||||
self::$commitCache[$key] = true;
|
||||
}
|
||||
|
||||
private static function getCache(
|
||||
PhabricatorRepository $repository,
|
||||
$commit_identifier) {
|
||||
|
||||
$key = self::getCacheKey($repository, $commit_identifier);
|
||||
return idx(self::$commitCache, $key, false);
|
||||
}
|
||||
|
||||
private static function getCacheKey(
|
||||
PhabricatorRepository $repository,
|
||||
$commit_identifier) {
|
||||
|
||||
return $repository->getID().':'.$commit_identifier;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* -( Git Implementation )------------------------------------------------- */
|
||||
|
||||
|
|
@ -249,6 +407,147 @@ final class PhabricatorRepositoryPullLocalDaemon
|
|||
}
|
||||
|
||||
|
||||
/**
|
||||
* @task git
|
||||
*/
|
||||
private static function executeGitDiscover(
|
||||
PhabricatorRepository $repository) {
|
||||
|
||||
list($remotes) = $repository->execxLocalCommand(
|
||||
'remote show -n origin');
|
||||
|
||||
$matches = null;
|
||||
if (!preg_match('/^\s*Fetch URL:\s*(.*?)\s*$/m', $remotes, $matches)) {
|
||||
throw new Exception(
|
||||
"Expected 'Fetch URL' in 'git remote show -n origin'.");
|
||||
}
|
||||
|
||||
self::executeGitverifySameOrigin(
|
||||
$matches[1],
|
||||
$repository->getRemoteURI(),
|
||||
$repository->getLocalPath());
|
||||
|
||||
list($stdout) = $repository->execxLocalCommand(
|
||||
'branch -r --verbose --no-abbrev');
|
||||
|
||||
$branches = DiffusionGitBranchQuery::parseGitRemoteBranchOutput(
|
||||
$stdout,
|
||||
$only_this_remote = DiffusionBranchInformation::DEFAULT_GIT_REMOTE);
|
||||
|
||||
$tracked_something = false;
|
||||
foreach ($branches as $name => $commit) {
|
||||
if (!$repository->shouldTrackBranch($name)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$tracked_something = true;
|
||||
|
||||
if (self::isKnownCommit($repository, $commit)) {
|
||||
continue;
|
||||
} else {
|
||||
self::executeGitDiscoverCommit($repository, $commit);
|
||||
}
|
||||
}
|
||||
|
||||
if (!$tracked_something) {
|
||||
$repo_name = $repository->getName();
|
||||
$repo_callsign = $repository->getCallsign();
|
||||
throw new Exception(
|
||||
"Repository r{$repo_callsign} '{$repo_name}' has no tracked branches! ".
|
||||
"Verify that your branch filtering settings are correct.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @task git
|
||||
*/
|
||||
private static function executeGitDiscoverCommit(
|
||||
PhabricatorRepository $repository,
|
||||
$commit) {
|
||||
|
||||
$discover = array($commit);
|
||||
$insert = array($commit);
|
||||
|
||||
$seen_parent = array();
|
||||
|
||||
while (true) {
|
||||
$target = array_pop($discover);
|
||||
list($parents) = $repository->execxLocalCommand(
|
||||
'log -n1 --pretty="%%P" %s',
|
||||
$target);
|
||||
$parents = array_filter(explode(' ', trim($parents)));
|
||||
foreach ($parents as $parent) {
|
||||
if (isset($seen_parent[$parent])) {
|
||||
// We end up in a loop here somehow when we parse Arcanist if we
|
||||
// don't do this. TODO: Figure out why and draw a pretty diagram
|
||||
// since it's not evident how parsing a DAG with this causes the
|
||||
// loop to stop terminating.
|
||||
continue;
|
||||
}
|
||||
$seen_parent[$parent] = true;
|
||||
if (!self::isKnownCommit($repository, $parent)) {
|
||||
$discover[] = $parent;
|
||||
$insert[] = $parent;
|
||||
}
|
||||
}
|
||||
if (empty($discover)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
while (true) {
|
||||
$target = array_pop($insert);
|
||||
list($epoch) = $repository->execxLocalCommand(
|
||||
'log -n1 --pretty="%%ct" %s',
|
||||
$target);
|
||||
$epoch = trim($epoch);
|
||||
|
||||
self::recordCommit($repository, $target, $epoch);
|
||||
|
||||
if (empty($insert)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @task git
|
||||
*/
|
||||
public static function executeGitVerifySameOrigin($remote, $expect, $where) {
|
||||
$remote_uri = PhabricatorRepository::newPhutilURIFromGitURI($remote);
|
||||
$expect_uri = PhabricatorRepository::newPhutilURIFromGitURI($expect);
|
||||
|
||||
$remote_path = $remote_uri->getPath();
|
||||
$expect_path = $expect_uri->getPath();
|
||||
|
||||
$remote_match = self::executeGitNormalizePath($remote_path);
|
||||
$expect_match = self::executeGitNormalizePath($expect_path);
|
||||
|
||||
if ($remote_match != $expect_match) {
|
||||
throw new Exception(
|
||||
"Working copy at '{$where}' has a mismatched origin URL. It has ".
|
||||
"origin URL '{$remote}' (with remote path '{$remote_path}'), but the ".
|
||||
"configured URL '{$expect}' (with remote path '{$expect_path}') is ".
|
||||
"expected. Refusing to proceed because this may indicate that the ".
|
||||
"working copy is actually some other repository.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @task git
|
||||
*/
|
||||
private static function executeGitNormalizePath($path) {
|
||||
// Strip away trailing "/" and ".git", so similar paths correctly match.
|
||||
|
||||
$path = rtrim($path, '/');
|
||||
$path = preg_replace('/\.git$/', '', $path);
|
||||
return $path;
|
||||
}
|
||||
|
||||
|
||||
/* -( Mercurial Implementation )------------------------------------------- */
|
||||
|
||||
|
||||
|
|
@ -306,4 +605,177 @@ final class PhabricatorRepositoryPullLocalDaemon
|
|||
}
|
||||
}
|
||||
|
||||
private static function executeHgDiscover(PhabricatorRepository $repository) {
|
||||
// NOTE: "--debug" gives us 40-character hashes.
|
||||
list($stdout) = $repository->execxLocalCommand('--debug branches');
|
||||
|
||||
$branches = ArcanistMercurialParser::parseMercurialBranches($stdout);
|
||||
$got_something = false;
|
||||
foreach ($branches as $name => $branch) {
|
||||
$commit = $branch['rev'];
|
||||
if (self::isKnownCommit($repository, $commit)) {
|
||||
continue;
|
||||
} else {
|
||||
self::executeHgDiscoverCommit($repository, $commit);
|
||||
$got_something = true;
|
||||
}
|
||||
}
|
||||
|
||||
return $got_something;
|
||||
}
|
||||
|
||||
private static function executeHgDiscoverCommit(
|
||||
PhabricatorRepository $repository,
|
||||
$commit) {
|
||||
|
||||
$discover = array($commit);
|
||||
$insert = array($commit);
|
||||
|
||||
$seen_parent = array();
|
||||
|
||||
// For all the new commits at the branch heads, walk backward until we find
|
||||
// only commits we've aleady seen.
|
||||
while (true) {
|
||||
$target = array_pop($discover);
|
||||
list($stdout) = $repository->execxLocalCommand(
|
||||
'parents --rev %s --template %s',
|
||||
$target,
|
||||
'{node}\n');
|
||||
$parents = array_filter(explode("\n", trim($stdout)));
|
||||
foreach ($parents as $parent) {
|
||||
if (isset($seen_parent[$parent])) {
|
||||
continue;
|
||||
}
|
||||
$seen_parent[$parent] = true;
|
||||
if (!self::isKnownCommit($repository, $parent)) {
|
||||
$discover[] = $parent;
|
||||
$insert[] = $parent;
|
||||
}
|
||||
}
|
||||
if (empty($discover)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
while (true) {
|
||||
$target = array_pop($insert);
|
||||
list($stdout) = $repository->execxLocalCommand(
|
||||
'log --rev %s --template %s',
|
||||
$target,
|
||||
'{date|rfc822date}');
|
||||
$epoch = strtotime($stdout);
|
||||
|
||||
self::recordCommit($repository, $target, $epoch);
|
||||
|
||||
if (empty($insert)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* -( Subversion Implementation )------------------------------------------ */
|
||||
|
||||
|
||||
private static function executeSvnDiscover(
|
||||
PhabricatorRepository $repository) {
|
||||
|
||||
$uri = self::executeSvnGetBaseSVNLogURI($repository);
|
||||
|
||||
list($xml) = $repository->execxRemoteCommand(
|
||||
'log --xml --quiet --limit 1 %s@HEAD',
|
||||
$uri);
|
||||
|
||||
$results = self::executeSvnParseLogXML($xml);
|
||||
$commit = head_key($results);
|
||||
$epoch = head($results);
|
||||
|
||||
if (self::isKnownCommit($repository, $commit)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
self::executeSvnDiscoverCommit($repository, $commit, $epoch);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static function executeSvnDiscoverCommit(
|
||||
PhabricatorRepository $repository,
|
||||
$commit,
|
||||
$epoch) {
|
||||
|
||||
$uri = self::executeSvnGetBaseSVNLogURI($repository);
|
||||
|
||||
$discover = array(
|
||||
$commit => $epoch,
|
||||
);
|
||||
$upper_bound = $commit;
|
||||
|
||||
$limit = 1;
|
||||
while ($upper_bound > 1 &&
|
||||
!self::isKnownCommit($repository, $upper_bound)) {
|
||||
// Find all the unknown commits on this path. Note that we permit
|
||||
// importing an SVN subdirectory rather than the entire repository, so
|
||||
// commits may be nonsequential.
|
||||
list($err, $xml, $stderr) = $repository->execRemoteCommand(
|
||||
' log --xml --quiet --limit %d %s@%d',
|
||||
$limit,
|
||||
$uri,
|
||||
$upper_bound - 1);
|
||||
if ($err) {
|
||||
if (preg_match('/(path|File) not found/', $stderr)) {
|
||||
// We've gone all the way back through history and this path was not
|
||||
// affected by earlier commits.
|
||||
break;
|
||||
} else {
|
||||
throw new Exception("svn log error #{$err}: {$stderr}");
|
||||
}
|
||||
}
|
||||
$discover += self::executeSvnParseLogXML($xml);
|
||||
|
||||
$upper_bound = min(array_keys($discover));
|
||||
|
||||
// Discover 2, 4, 8, ... 256 logs at a time. This allows us to initially
|
||||
// import large repositories fairly quickly, while pulling only as much
|
||||
// data as we need in the common case (when we've already imported the
|
||||
// repository and are just grabbing one commit at a time).
|
||||
$limit = min($limit * 2, 256);
|
||||
}
|
||||
|
||||
// NOTE: We do writes only after discovering all the commits so that we're
|
||||
// never left in a state where we've missed commits -- if the discovery
|
||||
// script terminates it can always resume and restore the import to a good
|
||||
// state. This is also why we sort the discovered commits so we can do
|
||||
// writes forward from the smallest one.
|
||||
|
||||
ksort($discover);
|
||||
foreach ($discover as $commit => $epoch) {
|
||||
self::recordCommit($repository, $commit, $epoch);
|
||||
}
|
||||
}
|
||||
|
||||
private static function executeSvnParseLogXML($xml) {
|
||||
$xml = phutil_utf8ize($xml);
|
||||
|
||||
$result = array();
|
||||
|
||||
$log = new SimpleXMLElement($xml);
|
||||
foreach ($log->logentry as $entry) {
|
||||
$commit = (int)$entry['revision'];
|
||||
$epoch = (int)strtotime((string)$entry->date[0]);
|
||||
$result[$commit] = $epoch;
|
||||
}
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
|
||||
private static function executeSvnGetBaseSVNLogURI(
|
||||
PhabricatorRepository $repository) {
|
||||
|
||||
$uri = $repository->getDetail('remote-uri');
|
||||
$subpath = $repository->getDetail('svn-subpath');
|
||||
|
||||
return $uri.$subpath;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,12 +6,20 @@
|
|||
|
||||
|
||||
|
||||
phutil_require_module('arcanist', 'repository/parser/mercurial');
|
||||
|
||||
phutil_require_module('phabricator', 'applications/diffusion/data/branch');
|
||||
phutil_require_module('phabricator', 'applications/diffusion/query/branch/git');
|
||||
phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
|
||||
phutil_require_module('phabricator', 'applications/repository/storage/commit');
|
||||
phutil_require_module('phabricator', 'applications/repository/storage/repository');
|
||||
phutil_require_module('phabricator', 'infrastructure/daemon/base');
|
||||
phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/event');
|
||||
phutil_require_module('phabricator', 'storage/queryfx');
|
||||
|
||||
phutil_require_module('phutil', 'error');
|
||||
phutil_require_module('phutil', 'filesystem');
|
||||
phutil_require_module('phutil', 'parser/argument/parser');
|
||||
phutil_require_module('phutil', 'utils');
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -16,10 +16,10 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
final class PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase
|
||||
final class PhabricatorRepositoryPullLocalDaemonTestCase
|
||||
extends PhabricatorTestCase {
|
||||
|
||||
public function testVerifySameGitOrigin() {
|
||||
public function testExecuteGitVerifySameOrigin() {
|
||||
$cases = array(
|
||||
array(
|
||||
'ssh://user@domain.com/path.git',
|
||||
|
|
@ -94,7 +94,7 @@ final class PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase
|
|||
|
||||
$ex = null;
|
||||
try {
|
||||
PhabricatorRepositoryGitCommitDiscoveryDaemon::verifySameGitOrigin(
|
||||
PhabricatorRepositoryPullLocalDaemon::executeGitverifySameOrigin(
|
||||
$remote,
|
||||
$config,
|
||||
'(a test case)');
|
||||
|
|
@ -6,8 +6,8 @@
|
|||
|
||||
|
||||
|
||||
phutil_require_module('phabricator', 'applications/repository/daemon/commitdiscovery/git');
|
||||
phutil_require_module('phabricator', 'applications/repository/daemon/pulllocal');
|
||||
phutil_require_module('phabricator', 'infrastructure/testing/testcase');
|
||||
|
||||
|
||||
phutil_require_source('PhabricatorRepositoryGitCommitDiscoveryDaemonTestCase.php');
|
||||
phutil_require_source('PhabricatorRepositoryPullLocalDaemonTestCase.php');
|
||||
Loading…
Reference in a new issue