diff --git a/bin/phd b/bin/phd new file mode 120000 index 0000000000..f1945da552 --- /dev/null +++ b/bin/phd @@ -0,0 +1 @@ +../scripts/daemon/phabricator_daemon_launcher.php \ No newline at end of file diff --git a/scripts/__init_script__.php b/scripts/__init_script__.php index a69ac9a5a8..7a67b3c587 100644 --- a/scripts/__init_script__.php +++ b/scripts/__init_script__.php @@ -25,8 +25,10 @@ if (!@constant('__LIBPHUTIL__')) { exit(1); } -if (!ini_get('date.timezone')) { - date_default_timezone_set('America/Los_Angeles'); -} - phutil_load_library(dirname(__FILE__).'/../src/'); + +phutil_require_module('phutil', 'symbols'); + +function __autoload($class) { + PhutilSymbolLoader::loadClass($class); +} diff --git a/scripts/daemon/phabricator_daemon_launcher.php b/scripts/daemon/phabricator_daemon_launcher.php new file mode 100755 index 0000000000..a2c418b09b --- /dev/null +++ b/scripts/daemon/phabricator_daemon_launcher.php @@ -0,0 +1,81 @@ +#!/usr/bin/env php +loadOneWhere( + 'callsign = %s', + $matches[1]); + if (!$repo) { + throw new Exception("Unknown repository!"); + } + $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere( + 'repositoryID = %d AND commitIdentifier = %s', + $repo->getID(), + $matches[2]); + if (!$commit) { + throw new Exception('Unknown commit.'); + } + + switch ($repo->getVersionControlSystem()) { + case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: + $worker = new PhabricatorRepositoryGitCommitChangeParserWorker( + $commit->getID()); + break; + case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: + $worker = new PhabricatorRepositorySvnCommitChangeParserWorker( + $commit->getID()); + break; + default: + throw new Exception("Unknown repository type!"); + } + + ExecFuture::pushEchoMode(true); + + $worker->doWork(); + + echo "Done.\n"; + + break; + case '--help': + case 'help': + default: + echo << + Parse a single commit. + +EOHELP; + exit(1); +} diff --git a/src/applications/diffusion/controller/home/__init__.php b/src/applications/diffusion/controller/home/__init__.php index 89f1fe41b5..c834c1e85e 100644 --- a/src/applications/diffusion/controller/home/__init__.php +++ b/src/applications/diffusion/controller/home/__init__.php @@ -7,12 +7,12 @@ phutil_require_module('phabricator', 'applications/diffusion/controller/base'); +phutil_require_module('phabricator', 'applications/diffusion/view/base'); phutil_require_module('phabricator', 'applications/repository/storage/commit'); phutil_require_module('phabricator', 'applications/repository/storage/repository'); phutil_require_module('phabricator', 'storage/queryfx'); phutil_require_module('phabricator', 'view/control/table'); phutil_require_module('phabricator', 'view/layout/panel'); -phutil_require_module('phabricator', 'view/utils'); phutil_require_module('phutil', 'markup'); phutil_require_module('phutil', 'utils'); diff --git a/src/applications/diffusion/controller/repository/__init__.php b/src/applications/diffusion/controller/repository/__init__.php index 6f5e15daaa..9aa13f1ef8 100644 --- a/src/applications/diffusion/controller/repository/__init__.php +++ b/src/applications/diffusion/controller/repository/__init__.php @@ -15,5 +15,7 @@ phutil_require_module('phabricator', 'applications/diffusion/view/browsetable'); phutil_require_module('phabricator', 'applications/diffusion/view/historytable'); phutil_require_module('phabricator', 'view/layout/panel'); +phutil_require_module('phutil', 'markup'); + phutil_require_source('DiffusionRepositoryController.php'); diff --git a/src/applications/diffusion/query/history/svn/__init__.php b/src/applications/diffusion/query/history/svn/__init__.php index c82a795dc9..4c14b8da0f 100644 --- a/src/applications/diffusion/query/history/svn/__init__.php +++ b/src/applications/diffusion/query/history/svn/__init__.php @@ -8,6 +8,8 @@ phutil_require_module('phabricator', 'applications/diffusion/data/pathchange'); phutil_require_module('phabricator', 'applications/diffusion/query/history/base'); +phutil_require_module('phabricator', 'applications/repository/storage/commit'); +phutil_require_module('phabricator', 'applications/repository/storage/commitdata'); phutil_require_module('phabricator', 'applications/repository/storage/repository'); phutil_require_module('phabricator', 'storage/queryfx'); diff --git a/src/applications/diffusion/view/historytable/__init__.php b/src/applications/diffusion/view/historytable/__init__.php index 5607901c5f..46ed27ee07 100644 --- a/src/applications/diffusion/view/historytable/__init__.php +++ b/src/applications/diffusion/view/historytable/__init__.php @@ -8,7 +8,6 @@ phutil_require_module('phabricator', 'applications/diffusion/view/base'); phutil_require_module('phabricator', 'view/control/table'); -phutil_require_module('phabricator', 'view/utils'); phutil_require_module('phutil', 'markup'); diff --git a/src/applications/repository/worker/commitchangeparser/svn/PhabricatorRepositorySvnCommitChangeParserWorker.php b/src/applications/repository/worker/commitchangeparser/svn/PhabricatorRepositorySvnCommitChangeParserWorker.php index b1a620f55e..5ce4d1ce94 100644 --- a/src/applications/repository/worker/commitchangeparser/svn/PhabricatorRepositorySvnCommitChangeParserWorker.php +++ b/src/applications/repository/worker/commitchangeparser/svn/PhabricatorRepositorySvnCommitChangeParserWorker.php @@ -516,7 +516,7 @@ class PhabricatorRepositorySvnCommitChangeParserWorker $parent = $repository_uri.$parent.'@'.$lookup['rawCommit']; $parent = escapeshellarg($parent); $parents[$parent] = true; - $path_mapping[$parent][] = $path; + $path_mapping[$parent][] = dirname($path); } $result_map = array(); @@ -592,21 +592,104 @@ class PhabricatorRepositorySvnCommitChangeParserWorker $rev = $info['rawCommit']; $path = $this->encodeSVNPath($path); - // TODO: This is a scalability nightmare. + $hashkey = md5($repository->getDetail('remote-uri').$path.'@'.$rev); - list($raw_xml) = execx( - 'svn --non-interactive --xml ls -R %s%s@%d', - $repository->getDetail('remote-uri'), - $path, - $rev); + // This method is quite horrible. The underlying challenge is that some + // commits in the Facebook repository are enormous, taking multiple hours + // to 'ls -R' out of the repository and producing XML files >1GB in size. + // If we try to SimpleXML them, the object exhausts available memory on a + // 64G machine. Instead, cache the XML output and then parse it line by line + // to limit space requirements. + + $cache_loc = sys_get_temp_dir().'/diffusion.'.$hashkey.'.svnls'; + if (!Filesystem::pathExists($cache_loc)) { + $tmp = new TempFile(); + execx( + 'svn --non-interactive --xml ls -R %s%s@%d > %s', + $repository->getDetail('remote-uri'), + $path, + $rev, + $tmp); + execx( + 'mv %s %s', + $tmp, + $cache_loc); + } + + $map = $this->parseRecursiveListFileData($cache_loc); + Filesystem::remove($cache_loc); + + return $map; + } + + private function parseRecursiveListFileData($file_path) { $map = array(); - $xml = new SimpleXMLElement($raw_xml); - foreach ($xml->list[0] as $entry) { - $key = (string)$entry->name; - $file_type = $this->getFileTypeFromSVNKind($entry['kind']); - $map[$key] = $file_type; + $mode = 'xml'; + $done = false; + $entry = null; + foreach (new LinesOfALargeFile($file_path) as $lno => $line) { + switch ($mode) { + case 'entry': + if ($line == '') { + $entry = implode('', $entry); + $pattern = '@^\s+kind="(file|dir)">'. + '(.*?)'. + '((.*?))?@'; + $matches = null; + if (!preg_match($pattern, $entry, $matches)) { + throw new Exception("Unable to parse entry!"); + } + $map[html_entity_decode($matches[2])] = + $this->getFileTypeFromSVNKind($matches[1]); + $mode = 'entry-or-end'; + } else { + $entry[] = $line; + } + break; + case 'entry-or-end': + if ($line == '') { + $done = true; + break 2; + } else if ($line == ' or