From b1e1b1f9bdf4474fa792695da29c0378e42e8297 Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 26 Sep 2011 11:07:38 -0700 Subject: [PATCH] Basic support for Mercurial in Diffusion Summary: Change import script plus almost all the view stuff. Still some rough edges but this seems to mostly work. Blame is currently unsupported but I think everything else works properly. Test Plan: Imported the hg repository itself. It doesn't immediately seem completely broken. Here are some screens: https://secure.phabricator.com/file/view/PHID-FILE-1438b71cc7c4a2eb4569/ https://secure.phabricator.com/file/view/PHID-FILE-3cec4f72f39e7de2d041/ https://secure.phabricator.com/file/view/PHID-FILE-2ea4883f160e8e5098f9/ https://secure.phabricator.com/file/view/PHID-FILE-35f751a36ebf65399ade/ All the parsers were able to churn through it without errors. Ran the new "reparse.php" script in various one-commit and repository modes. Browsed/imported some git repos for good measure. NOTE: The hg repository is only 15,000 commits and around 1,000 files. Performance is okay but hg doesn't provide performant, native APIs to get some data efficiently so we have to do some dumb stuff. If some of these interfaces are cripplingly slow or whatever, let me know and we can start bundling some Mercurial extensions with Arcanist. Reviewers: Makinde, jungejason, nh, tuomaspelkonen, aran Reviewed By: Makinde CC: aran, Makinde, epriestley Differential Revision: 960 --- src/__phutil_library_map__.php | 19 +- .../commit/DiffusionCommitController.php | 1 + .../branch/base/DiffusionBranchQuery.php | 3 + .../DiffusionMercurialBranchQuery.php | 40 ++ .../query/branch/mercurial/__init__.php | 15 + .../browse/base/DiffusionBrowseQuery.php | 3 + .../DiffusionMercurialBrowseQuery.php | 95 +++++ .../query/browse/mercurial/__init__.php | 14 + .../query/diff/base/DiffusionDiffQuery.php | 3 + .../mercurial/DiffusionMercurialDiffQuery.php | 56 +++ .../query/diff/mercurial/__init__.php | 15 + .../base/DiffusionFileContentQuery.php | 3 + .../DiffusionMercurialFileContentQuery.php | 47 +++ .../query/filecontent/mercurial/__init__.php | 13 + .../history/base/DiffusionHistoryQuery.php | 75 ++++ .../diffusion/query/history/base/__init__.php | 7 + .../history/git/DiffusionGitHistoryQuery.php | 62 +-- .../diffusion/query/history/git/__init__.php | 6 - .../DiffusionMercurialHistoryQuery.php | 45 +++ .../query/history/mercurial/__init__.php | 15 + .../base/DiffusionLastModifiedQuery.php | 3 + .../DiffusionMercurialLastModifiedQuery.php | 50 +++ .../query/lastmodified/mercurial/__init__.php | 16 + .../request/base/DiffusionRequest.php | 3 + .../mercurial/DiffusionMercurialRequest.php | 81 ++++ .../diffusion/request/mercurial/__init__.php | 12 + ...itoryMercurialCommitChangeParserWorker.php | 354 ++++++++++++++++++ .../commitchangeparser/mercurial/__init__.php | 19 + ...rRepositorySvnCommitChangeParserWorker.php | 2 + 29 files changed, 1007 insertions(+), 70 deletions(-) create mode 100644 src/applications/diffusion/query/branch/mercurial/DiffusionMercurialBranchQuery.php create mode 100644 src/applications/diffusion/query/branch/mercurial/__init__.php create mode 100644 src/applications/diffusion/query/browse/mercurial/DiffusionMercurialBrowseQuery.php create mode 100644 src/applications/diffusion/query/browse/mercurial/__init__.php create mode 100644 src/applications/diffusion/query/diff/mercurial/DiffusionMercurialDiffQuery.php create mode 100644 src/applications/diffusion/query/diff/mercurial/__init__.php create mode 100644 src/applications/diffusion/query/filecontent/mercurial/DiffusionMercurialFileContentQuery.php create mode 100644 src/applications/diffusion/query/filecontent/mercurial/__init__.php create mode 100644 src/applications/diffusion/query/history/mercurial/DiffusionMercurialHistoryQuery.php create mode 100644 src/applications/diffusion/query/history/mercurial/__init__.php create mode 100644 src/applications/diffusion/query/lastmodified/mercurial/DiffusionMercurialLastModifiedQuery.php create mode 100644 src/applications/diffusion/query/lastmodified/mercurial/__init__.php create mode 100644 src/applications/diffusion/request/mercurial/DiffusionMercurialRequest.php create mode 100644 src/applications/diffusion/request/mercurial/__init__.php create mode 100644 src/applications/repository/worker/commitchangeparser/mercurial/PhabricatorRepositoryMercurialCommitChangeParserWorker.php create mode 100644 src/applications/repository/worker/commitchangeparser/mercurial/__init__.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 02b470837f..50e17cecc0 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -261,6 +261,13 @@ phutil_register_library_map(array( 'DiffusionHomeController' => 'applications/diffusion/controller/home', 'DiffusionLastModifiedController' => 'applications/diffusion/controller/lastmodified', 'DiffusionLastModifiedQuery' => 'applications/diffusion/query/lastmodified/base', + 'DiffusionMercurialBranchQuery' => 'applications/diffusion/query/branch/mercurial', + 'DiffusionMercurialBrowseQuery' => 'applications/diffusion/query/browse/mercurial', + 'DiffusionMercurialDiffQuery' => 'applications/diffusion/query/diff/mercurial', + 'DiffusionMercurialFileContentQuery' => 'applications/diffusion/query/filecontent/mercurial', + 'DiffusionMercurialHistoryQuery' => 'applications/diffusion/query/history/mercurial', + 'DiffusionMercurialLastModifiedQuery' => 'applications/diffusion/query/lastmodified/mercurial', + 'DiffusionMercurialRequest' => 'applications/diffusion/request/mercurial', 'DiffusionPathChange' => 'applications/diffusion/data/pathchange', 'DiffusionPathChangeQuery' => 'applications/diffusion/query/pathchange/base', 'DiffusionPathCompleteController' => 'applications/diffusion/controller/pathcomplete', @@ -582,6 +589,7 @@ phutil_register_library_map(array( 'PhabricatorRepositoryGitHubNotification' => 'applications/repository/storage/githubnotification', 'PhabricatorRepositoryGitHubPostReceiveController' => 'applications/repository/controller/github-post-receive', 'PhabricatorRepositoryListController' => 'applications/repository/controller/list', + 'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/mercurial', 'PhabricatorRepositoryMercurialCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/mercurial', 'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/mercurial', 'PhabricatorRepositoryMercurialPullDaemon' => 'applications/repository/daemon/mercurialpull', @@ -927,6 +935,13 @@ phutil_register_library_map(array( 'DiffusionHistoryTableView' => 'DiffusionView', 'DiffusionHomeController' => 'DiffusionController', 'DiffusionLastModifiedController' => 'DiffusionController', + 'DiffusionMercurialBranchQuery' => 'DiffusionBranchQuery', + 'DiffusionMercurialBrowseQuery' => 'DiffusionBrowseQuery', + 'DiffusionMercurialDiffQuery' => 'DiffusionDiffQuery', + 'DiffusionMercurialFileContentQuery' => 'DiffusionFileContentQuery', + 'DiffusionMercurialHistoryQuery' => 'DiffusionHistoryQuery', + 'DiffusionMercurialLastModifiedQuery' => 'DiffusionLastModifiedQuery', + 'DiffusionMercurialRequest' => 'DiffusionRequest', 'DiffusionPathCompleteController' => 'DiffusionController', 'DiffusionPathQueryTestCase' => 'PhabricatorTestCase', 'DiffusionPathValidateController' => 'DiffusionController', @@ -1188,10 +1203,8 @@ phutil_register_library_map(array( 'PhabricatorRepositoryGitHubNotification' => 'PhabricatorRepositoryDAO', 'PhabricatorRepositoryGitHubPostReceiveController' => 'PhabricatorRepositoryController', 'PhabricatorRepositoryListController' => 'PhabricatorRepositoryController', -<<<<<<< HEAD + 'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker', 'PhabricatorRepositoryMercurialCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon', -======= ->>>>>>> Add a Mercurial message parser 'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker', 'PhabricatorRepositoryMercurialPullDaemon' => 'PhabricatorRepositoryPullLocalDaemon', 'PhabricatorRepositoryPullLocalDaemon' => 'PhabricatorRepositoryDaemon', diff --git a/src/applications/diffusion/controller/commit/DiffusionCommitController.php b/src/applications/diffusion/controller/commit/DiffusionCommitController.php index 1d8566656d..3a0125d74e 100644 --- a/src/applications/diffusion/controller/commit/DiffusionCommitController.php +++ b/src/applications/diffusion/controller/commit/DiffusionCommitController.php @@ -172,6 +172,7 @@ class DiffusionCommitController extends DiffusionController { $vcs_supports_directory_changes = true; break; case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: + case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: $vcs_supports_directory_changes = false; break; default: diff --git a/src/applications/diffusion/query/branch/base/DiffusionBranchQuery.php b/src/applications/diffusion/query/branch/base/DiffusionBranchQuery.php index 811916f0cd..a7b598ecd7 100644 --- a/src/applications/diffusion/query/branch/base/DiffusionBranchQuery.php +++ b/src/applications/diffusion/query/branch/base/DiffusionBranchQuery.php @@ -33,6 +33,9 @@ abstract class DiffusionBranchQuery { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $class = 'DiffusionGitBranchQuery'; break; + case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: + $class = 'DiffusionMercurialBranchQuery'; + break; default: throw new Exception("Unsupported VCS!"); } diff --git a/src/applications/diffusion/query/branch/mercurial/DiffusionMercurialBranchQuery.php b/src/applications/diffusion/query/branch/mercurial/DiffusionMercurialBranchQuery.php new file mode 100644 index 0000000000..1e058e8b56 --- /dev/null +++ b/src/applications/diffusion/query/branch/mercurial/DiffusionMercurialBranchQuery.php @@ -0,0 +1,40 @@ +getRequest(); + $repository = $drequest->getRepository(); + + list($stdout) = $repository->execxLocalCommand( + 'branches'); + $branch_info = ArcanistMercurialParser::parseMercurialBranches($stdout); + + $branches = array(); + foreach ($branch_info as $name => $info) { + $branch = new DiffusionBranchInformation(); + $branch->setName($name); + $branch->setHeadCommitIdentifier($info['rev']); + $branches[] = $branch; + } + + return $branches; + } + +} diff --git a/src/applications/diffusion/query/branch/mercurial/__init__.php b/src/applications/diffusion/query/branch/mercurial/__init__.php new file mode 100644 index 0000000000..290916e281 --- /dev/null +++ b/src/applications/diffusion/query/branch/mercurial/__init__.php @@ -0,0 +1,15 @@ +getRequest(); + $repository = $drequest->getRepository(); + + $path = $drequest->getPath(); + $commit = $drequest->getCommit(); + + // TODO: This is a really really awful mess but Mercurial doesn't offer + // an equivalent of "git ls-files -- directory". If it's any comfort, this + // is what "hgweb" does too, see: + // + // http://selenic.com/repo/hg/file/91dc8878f888/mercurial/hgweb/webcommands.py#l320 + // + // derp derp derp derp + // + // Anyway, figure out what's in this path by applying massive amounts + // of brute force. + + list($entire_manifest) = $repository->execxLocalCommand( + 'manifest --rev %s', + $commit); + $entire_manifest = explode("\n", $entire_manifest); + + $results = array(); + + $match_against = trim($path, '/'); + $match_len = strlen($match_against); + + // For the root, don't trim. For other paths, trim the "/" after we match. + // We need this because Mercurial's canonical paths have no leading "/", + // but ours do. + $trim_len = $match_len ? $match_len + 1 : 0; + + foreach ($entire_manifest as $path) { + if (strncmp($path, $match_against, $match_len)) { + continue; + } + if (!strlen($path)) { + continue; + } + $remainder = substr($path, $trim_len); + if (!strlen($remainder)) { + // There is a file with this exact name in the manifest, so clearly + // it's a file. + $this->reason = self::REASON_IS_FILE; + return array(); + } + $parts = explode('/', $remainder); + if (count($parts) == 1) { + $type = DifferentialChangeType::FILE_NORMAL; + } else { + $type = DifferentialChangeType::FILE_DIRECTORY; + } + $results[reset($parts)] = $type; + } + + foreach ($results as $key => $type) { + $result = new DiffusionRepositoryPath(); + $result->setPath($key); + $result->setFileType($type); + + $results[$key] = $result; + } + + if (empty($results)) { + // TODO: Detect "deleted" by issuing "hg log"? + + $this->reason = self::REASON_IS_NONEXISTENT; + } + + + return $results; + } + +} diff --git a/src/applications/diffusion/query/browse/mercurial/__init__.php b/src/applications/diffusion/query/browse/mercurial/__init__.php new file mode 100644 index 0000000000..3bf1099ab8 --- /dev/null +++ b/src/applications/diffusion/query/browse/mercurial/__init__.php @@ -0,0 +1,14 @@ +getRequest(); + $repository = $drequest->getRepository(); + + $effective_commit = $this->getEffectiveCommit(); + if (!$effective_commit) { + return null; + } + // TODO: This side effect is kind of skethcy. + $drequest->setCommit($effective_commit); + + $path = $drequest->getPath(); + + list($raw_diff) = $repository->execxLocalCommand( + 'diff -U %d --git --change %s -- %s', + 65535, + $effective_commit, + $path); + + $parser = new ArcanistDiffParser(); + $parser->setDetectBinaryFiles(true); + $changes = $parser->parseDiff($raw_diff); + + $diff = DifferentialDiff::newFromRawChanges($changes); + $changesets = $diff->getChangesets(); + $changeset = reset($changesets); + + $this->renderingReference = + $drequest->getBranchURIComponent($drequest->getBranch()). + $drequest->getPath().';'. + $drequest->getCommit(); + + return $changeset; + } + +} diff --git a/src/applications/diffusion/query/diff/mercurial/__init__.php b/src/applications/diffusion/query/diff/mercurial/__init__.php new file mode 100644 index 0000000000..8daa080d53 --- /dev/null +++ b/src/applications/diffusion/query/diff/mercurial/__init__.php @@ -0,0 +1,15 @@ +getRequest(); + + $repository = $drequest->getRepository(); + $path = $drequest->getPath(); + $commit = $drequest->getCommit(); + + list($corpus) = $repository->execxLocalCommand( + 'cat --rev %s -- %s', + $commit, + $path); + + $file_content = new DiffusionFileContent(); + $file_content->setCorpus($corpus); + + return $file_content; + } + + + protected function tokenizeLine($line) { + // TODO: Support blame. + throw new Exception( + "Diffusion does not currently support blame for Mercurial."); + } + +} diff --git a/src/applications/diffusion/query/filecontent/mercurial/__init__.php b/src/applications/diffusion/query/filecontent/mercurial/__init__.php new file mode 100644 index 0000000000..1d6c8223e2 --- /dev/null +++ b/src/applications/diffusion/query/filecontent/mercurial/__init__.php @@ -0,0 +1,13 @@ +getRequest(); + $repository = $drequest->getRepository(); + + $path = $drequest->getPath(); + + $commits = id(new PhabricatorRepositoryCommit())->loadAllWhere( + 'repositoryID = %d AND commitIdentifier IN (%Ls)', + $repository->getID(), + $identifiers); + $commits = mpull($commits, null, 'getCommitIdentifier'); + + if (!$commits) { + return array(); + } + + $commit_data = id(new PhabricatorRepositoryCommitData())->loadAllWhere( + 'commitID in (%Ld)', + mpull($commits, 'getID')); + $commit_data = mpull($commit_data, null, 'getCommitID'); + + $conn_r = $repository->establishConnection('r'); + + $path_normal = DiffusionPathIDQuery::normalizePath($path); + $paths = queryfx_all( + $conn_r, + 'SELECT id, path FROM %T WHERE path IN (%Ls)', + PhabricatorRepository::TABLE_PATH, + array($path_normal)); + $paths = ipull($paths, 'id', 'path'); + $path_id = idx($paths, $path_normal); + + $path_changes = queryfx_all( + $conn_r, + 'SELECT * FROM %T WHERE commitID IN (%Ld) AND pathID = %d', + PhabricatorRepository::TABLE_PATHCHANGE, + mpull($commits, 'getID'), + $path_id); + $path_changes = ipull($path_changes, null, 'commitID'); + + $history = array(); + foreach ($identifiers as $identifier) { + $item = new DiffusionPathChange(); + $item->setCommitIdentifier($identifier); + $commit = idx($commits, $identifier); + if ($commit) { + $item->setCommit($commit); + $data = idx($commit_data, $commit->getID()); + if ($data) { + $item->setCommitData($data); + } + $change = idx($path_changes, $commit->getID()); + if ($change) { + $item->setChangeType($change['changeType']); + $item->setFileType($change['fileType']); + } + } + $history[] = $item; + } + + return $history; + } + } diff --git a/src/applications/diffusion/query/history/base/__init__.php b/src/applications/diffusion/query/history/base/__init__.php index 010bb2a494..96499f2f3e 100644 --- a/src/applications/diffusion/query/history/base/__init__.php +++ b/src/applications/diffusion/query/history/base/__init__.php @@ -6,9 +6,16 @@ +phutil_require_module('phabricator', 'applications/diffusion/data/pathchange'); +phutil_require_module('phabricator', 'applications/diffusion/query/pathid/base'); phutil_require_module('phabricator', 'applications/repository/constants/repositorytype'); +phutil_require_module('phabricator', 'applications/repository/storage/commit'); +phutil_require_module('phabricator', 'applications/repository/storage/commitdata'); +phutil_require_module('phabricator', 'applications/repository/storage/repository'); +phutil_require_module('phabricator', 'storage/queryfx'); phutil_require_module('phutil', 'symbols'); +phutil_require_module('phutil', 'utils'); phutil_require_source('DiffusionHistoryQuery.php'); diff --git a/src/applications/diffusion/query/history/git/DiffusionGitHistoryQuery.php b/src/applications/diffusion/query/history/git/DiffusionGitHistoryQuery.php index 44d23695b9..bf95cfe610 100644 --- a/src/applications/diffusion/query/history/git/DiffusionGitHistoryQuery.php +++ b/src/applications/diffusion/query/history/git/DiffusionGitHistoryQuery.php @@ -43,67 +43,7 @@ final class DiffusionGitHistoryQuery extends DiffusionHistoryQuery { $hashes = explode("\n", $stdout); $hashes = array_filter($hashes); - $commits = array(); - $commit_data = array(); - $path_changes = array(); - - $conn_r = $repository->establishConnection('r'); - - if ($hashes) { - $commits = id(new PhabricatorRepositoryCommit())->loadAllWhere( - 'repositoryID = %d AND commitIdentifier IN (%Ls)', - $repository->getID(), - $hashes); - $commits = mpull($commits, null, 'getCommitIdentifier'); - if ($commits) { - $commit_data = id(new PhabricatorRepositoryCommitData())->loadAllWhere( - 'commitID in (%Ld)', - mpull($commits, 'getID')); - $commit_data = mpull($commit_data, null, 'getCommitID'); - } - - if ($commits) { - $path_normal = '/'.trim($path, '/'); - $paths = queryfx_all( - $conn_r, - 'SELECT id, path FROM %T WHERE path IN (%Ls)', - PhabricatorRepository::TABLE_PATH, - array($path_normal)); - $paths = ipull($paths, 'id', 'path'); - $path_id = idx($paths, $path_normal); - - $path_changes = queryfx_all( - $conn_r, - 'SELECT * FROM %T WHERE commitID IN (%Ld) AND pathID = %d', - PhabricatorRepository::TABLE_PATHCHANGE, - mpull($commits, 'getID'), - $path_id); - $path_changes = ipull($path_changes, null, 'commitID'); - } - } - - - $history = array(); - foreach ($hashes as $hash) { - $item = new DiffusionPathChange(); - $item->setCommitIdentifier($hash); - $commit = idx($commits, $hash); - if ($commit) { - $item->setCommit($commit); - $data = idx($commit_data, $commit->getID()); - if ($data) { - $item->setCommitData($data); - } - $change = idx($path_changes, $commit->getID()); - if ($change) { - $item->setChangeType($change['changeType']); - $item->setFileType($change['fileType']); - } - } - $history[] = $item; - } - - return $history; + return $this->loadHistoryForCommitIdentifiers($hashes); } } diff --git a/src/applications/diffusion/query/history/git/__init__.php b/src/applications/diffusion/query/history/git/__init__.php index 8d20e55a91..1094fb4883 100644 --- a/src/applications/diffusion/query/history/git/__init__.php +++ b/src/applications/diffusion/query/history/git/__init__.php @@ -6,15 +6,9 @@ -phutil_require_module('phabricator', 'applications/diffusion/data/pathchange'); phutil_require_module('phabricator', 'applications/diffusion/query/history/base'); -phutil_require_module('phabricator', 'applications/repository/storage/commit'); -phutil_require_module('phabricator', 'applications/repository/storage/commitdata'); -phutil_require_module('phabricator', 'applications/repository/storage/repository'); -phutil_require_module('phabricator', 'storage/queryfx'); phutil_require_module('phutil', 'future/exec'); -phutil_require_module('phutil', 'utils'); phutil_require_source('DiffusionGitHistoryQuery.php'); diff --git a/src/applications/diffusion/query/history/mercurial/DiffusionMercurialHistoryQuery.php b/src/applications/diffusion/query/history/mercurial/DiffusionMercurialHistoryQuery.php new file mode 100644 index 0000000000..2e7ad308b8 --- /dev/null +++ b/src/applications/diffusion/query/history/mercurial/DiffusionMercurialHistoryQuery.php @@ -0,0 +1,45 @@ +getRequest(); + + $repository = $drequest->getRepository(); + $path = $drequest->getPath(); + $commit_hash = $drequest->getCommit(); + + $path = DiffusionPathIDQuery::normalizePath($path); + + list($stdout) = $repository->execxLocalCommand( + 'log --template %s --limit %d --branch %s --rev %s:0 -- %s', + '{node}\\n', + ($this->getOffset() + $this->getLimit()), // No '--skip' in Mercurial. + $drequest->getBranch(), + $commit_hash, + nonempty(ltrim($path, '/'), '.')); + + $hashes = explode("\n", $stdout); + $hashes = array_filter($hashes); + $hashes = array_slice($hashes, $this->getOffset()); + + return $this->loadHistoryForCommitIdentifiers($hashes); + } + +} diff --git a/src/applications/diffusion/query/history/mercurial/__init__.php b/src/applications/diffusion/query/history/mercurial/__init__.php new file mode 100644 index 0000000000..b68c46da97 --- /dev/null +++ b/src/applications/diffusion/query/history/mercurial/__init__.php @@ -0,0 +1,15 @@ +getRequest(); + $repository = $drequest->getRepository(); + + $path = $drequest->getPath(); + + // TODO: Share some of this with History query. + list($hash) = $repository->execxLocalCommand( + 'log --template %s --limit 1 --branch %s --rev %s:0 -- %s', + '{node}\\n', + $drequest->getBranch(), + $drequest->getCommit(), + nonempty(ltrim($path, '/'), '.')); + $hash = trim($hash); + + $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere( + 'repositoryID = %d AND commitIdentifier = %s', + $repository->getID(), + $hash); + if ($commit) { + $commit_data = id(new PhabricatorRepositoryCommitData())->loadOneWhere( + 'commitID = %d', + $commit->getID()); + } + + return array($commit, $commit_data); + } + +} diff --git a/src/applications/diffusion/query/lastmodified/mercurial/__init__.php b/src/applications/diffusion/query/lastmodified/mercurial/__init__.php new file mode 100644 index 0000000000..5a187f77c3 --- /dev/null +++ b/src/applications/diffusion/query/lastmodified/mercurial/__init__.php @@ -0,0 +1,16 @@ +path; + $parts = explode('/', $path); + + $branch = array_shift($parts); + if ($branch != ':') { + $this->branch = $this->decodeBranchName($branch); + } + + foreach ($parts as $key => $part) { + if ($part == '..') { + unset($parts[$key]); + } + } + + $this->path = implode('/', $parts); + } + + public function getBranch() { + if ($this->branch) { + return $this->branch; + } + if ($this->repository) { + return $this->repository->getDetail('default-branch', 'default'); + } + throw new Exception("Unable to determine branch!"); + } + + public function getUriPath() { + return '/diffusion/'.$this->getCallsign().'/browse/'. + $this->getBranchURIComponent($this->branch).$this->path; + } + + public function getCommit() { + if ($this->commit) { + return $this->commit; + } + return $this->getBranch(); + } + + public function getStableCommitName() { + return substr($this->stableCommitName, 0, 16); + } + + public function getBranchURIComponent($branch) { + return $this->encodeBranchName($branch).'/'; + } + + private function decodeBranchName($branch) { + return str_replace(':', '/', $branch); + } + + private function encodeBranchName($branch) { + return str_replace('/', ':', $branch); + } + +} diff --git a/src/applications/diffusion/request/mercurial/__init__.php b/src/applications/diffusion/request/mercurial/__init__.php new file mode 100644 index 0000000000..32af21a2de --- /dev/null +++ b/src/applications/diffusion/request/mercurial/__init__.php @@ -0,0 +1,12 @@ +getCallsign().$commit->getCommitIdentifier(); + echo "Parsing {$full_name}...\n"; + if ($this->isBadCommit($full_name)) { + echo "This commit is marked bad!\n"; + return; + } + + list($stdout) = $repository->execxLocalCommand( + 'status -C --change %s', + $commit->getCommitIdentifier()); + $status = ArcanistMercurialParser::parseMercurialStatusDetails($stdout); + + $common_attributes = array( + 'repositoryID' => $repository->getID(), + 'commitID' => $commit->getID(), + 'commitSequence' => $commit->getEpoch(), + ); + + $changes = array(); + + // Like Git, Mercurial doesn't track directories directly. We need to infer + // directory creation and removal by observing file creation and removal + // and testing if the directories in question are previously empty (thus, + // created) or subsequently empty (thus, removed). + $maybe_new_directories = array(); + $maybe_del_directories = array(); + $all_directories = array(); + + // Parse the basic information from "hg status", which shows files that + // were directly affected by the change. + foreach ($status as $path => $path_info) { + $path = '/'.$path; + $flags = $path_info['flags']; + $change_target = $path_info['from'] ? '/'.$path_info['from'] : null; + + $changes[$path] = array( + 'path' => $path, + 'isDirect' => true, + + 'targetPath' => $change_target, + 'targetCommitID' => $change_target ? $commit->getID() : null, + + // We're going to fill these in shortly. + 'changeType' => null, + 'fileType' => null, + + 'flags' => $flags, + ) + $common_attributes; + + if ($flags & ArcanistRepositoryAPI::FLAG_ADDED) { + $maybe_new_directories[] = dirname($path); + } else if ($flags & ArcanistRepositoryAPI::FLAG_DELETED) { + $maybe_del_directories[] = dirname($path); + } + $all_directories[] = dirname($path); + } + + // Add change information for each source path which doesn't appear in the + // status. These files were copied, but were not modified. We also know they + // must exist. + foreach ($changes as $path => $change) { + $from = $change['targetPath']; + if ($from && empty($changes[$from])) { + $changes[$from] = array( + 'path' => $from, + 'isDirect' => false, + + 'targetPath' => null, + 'targetCommitID' => null, + + 'changeType' => DifferentialChangeType::TYPE_COPY_AWAY, + 'fileType' => null, + + 'flags' => 0, + ) + $common_attributes; + } + } + + $away = array(); + foreach ($changes as $path => $change) { + if ($path['targetPath']) { + $away[$path['targetPath']][] = $path; + } + } + + // Now that we have all the direct changes, figure out change types. + foreach ($changes as $path => $change) { + $flags = $change['flags']; + $from = $change['targetPath']; + if ($from) { + $target = $changes[$from]; + } else { + $target = null; + } + + if ($flags & ArcanistRepositoryAPI::FLAG_ADDED) { + if ($target) { + if ($target['flags'] & ArcanistRepositoryAPI::FLAG_DELETED) { + $change_type = DifferentialChangeType::TYPE_MOVE_HERE; + } else { + $change_type = DifferentialChangeType::TYPE_COPY_HERE; + } + } else { + $change_type = DifferentialChangeType::TYPE_ADD; + } + } else if ($flags & ArcanistRepositoryAPI::FLAG_DELETED) { + if (isset($away[$path])) { + if (count($away[$path]) > 1) { + $change_type = DifferentialChangeType::TYPE_MULTICOPY; + } else { + $change_type = DifferentialChangeType::TYPE_MOVE_AWAY; + } + } else { + $change_type = DifferentialChangeType::TYPE_DELETE; + } + } else { + if (isset($away[$path])) { + $change_type = DifferentialChangeType::TYPE_COPY_AWAY; + } else { + $change_type = DifferentialChangeType::TYPE_CHANGE; + } + } + + $changes[$path]['changeType'] = $change_type; + } + + // Go through all the affected directories and identify any which were + // actually added or deleted. + $dir_status = array(); + + foreach ($maybe_del_directories as $dir) { + $exists = false; + foreach (DiffusionPathIDQuery::expandPathToRoot($dir) as $path) { + if (isset($dir_status[$path])) { + break; + } + + // If we know some child exists, we know this path exists. If we don't + // know that a child exists, test if this directory still exists. + if (!$exists) { + $exists = $this->mercurialPathExists( + $repository, + $path, + $commit->getCommitIdentifier()); + } + + if ($exists) { + $dir_status[$path] = DifferentialChangeType::TYPE_CHILD; + } else { + $dir_status[$path] = DifferentialChangeType::TYPE_DELETE; + } + } + } + + list($stdout) = $repository->execxLocalCommand( + 'parents --rev %s --style default', + $commit->getCommitIdentifier()); + $parents = ArcanistMercurialParser::parseMercurialLog($stdout); + $parent = reset($parents); + if ($parent) { + // TODO: We should expand this to a full 40-character hash using "hg id". + $parent = $parent['rev']; + } + foreach ($maybe_new_directories as $dir) { + $exists = false; + foreach (DiffusionPathIDQuery::expandPathToRoot($dir) as $path) { + if (isset($dir_status[$path])) { + break; + } + if (!$exists) { + if ($parent) { + $exists = $this->mercurialPathExists($repository, $path, $parent); + } else { + $exists = false; + } + } + if ($exists) { + $dir_status[$path] = DifferentialChangeType::TYPE_CHILD; + } else { + $dir_status[$path] = DifferentialChangeType::TYPE_ADD; + } + } + } + + foreach ($all_directories as $dir) { + foreach (DiffusionPathIDQuery::expandPathToRoot($dir) as $path) { + if (isset($dir_status[$path])) { + break; + } + $dir_status[$path] = DifferentialChangeType::TYPE_CHILD; + } + } + + // Merge all the directory statuses into the path statuses. + foreach ($dir_status as $path => $status) { + if (isset($changes[$path])) { + // TODO: The UI probably doesn't handle any of these cases with + // terrible elegance, but they are exceedingly rare. + + $existing_type = $changes[$path]['changeType']; + if ($existing_type == DifferentialChangeType::TYPE_DELETE) { + // This change removes a file, replaces it with a directory, and then + // adds children of that directory. Mark it as a "change" instead, + // and make the type a directory. + $changes[$path]['fileType'] = DifferentialChangeType::FILE_DIRECTORY; + $changes[$path]['changeType'] = DifferentialChangeType::TYPE_CHANGE; + } else if ($existing_type == DifferentialChangeType::TYPE_MOVE_AWAY || + $existing_type == DifferentialChangeType::TYPE_MULTICOPY) { + // This change moves or copies a file, replaces it with a directory, + // and then adds children to that directory. Mark it as "copy away" + // instead of whatever it was, and make the type a directory. + $changes[$path]['fileType'] = DifferentialChangeType::FILE_DIRECTORY; + $changes[$path]['changeType'] + = DifferentialChangeType::TYPE_COPY_AWAY; + } else if ($existing_type == DifferentialChangeType::TYPE_ADD) { + // This change removes a diretory and replaces it with a file. Mark + // it as "change" instead of "add". + $changes[$path]['changeType'] = DifferentialChangeType::TYPE_CHANGE; + } + + continue; + } + + $changes[$path] = array( + 'path' => $path, + 'isDirect' => ($status == DifferentialChangeType::TYPE_CHILD) + ? false + : true, + 'fileType' => DifferentialChangeType::FILE_DIRECTORY, + 'changeType' => $status, + + 'targetPath' => null, + 'targetCommitID' => null, + ) + $common_attributes; + } + + // TODO: use "hg diff --git" to figure out which files are symlinks. + foreach ($changes as $path => $change) { + if (empty($change['fileType'])) { + $changes[$path]['fileType'] = DifferentialChangeType::FILE_NORMAL; + } + } + + $all_paths = array(); + foreach ($changes as $path => $change) { + $all_paths[$path] = true; + if ($change['targetPath']) { + $all_paths[$change['targetPath']] = true; + } + } + + $path_map = $this->lookupOrCreatePaths(array_keys($all_paths)); + + foreach ($changes as $key => $change) { + $changes[$key]['pathID'] = $path_map[$change['path']]; + if ($change['targetPath']) { + $changes[$key]['targetPathID'] = $path_map[$change['targetPath']]; + } else { + $changes[$key]['targetPathID'] = null; + } + } + + $conn_w = $repository->establishConnection('w'); + + $changes_sql = array(); + foreach ($changes as $change) { + $values = array( + (int)$change['repositoryID'], + (int)$change['pathID'], + (int)$change['commitID'], + $change['targetPathID'] + ? (int)$change['targetPathID'] + : 'null', + $change['targetCommitID'] + ? (int)$change['targetCommitID'] + : 'null', + (int)$change['changeType'], + (int)$change['fileType'], + (int)$change['isDirect'], + (int)$change['commitSequence'], + ); + $changes_sql[] = '('.implode(', ', $values).')'; + } + + queryfx( + $conn_w, + 'DELETE FROM %T WHERE commitID = %d', + PhabricatorRepository::TABLE_PATHCHANGE, + $commit->getID()); + foreach (array_chunk($changes_sql, 256) as $sql_chunk) { + queryfx( + $conn_w, + 'INSERT INTO %T + (repositoryID, pathID, commitID, targetPathID, targetCommitID, + changeType, fileType, isDirect, commitSequence) + VALUES %Q', + PhabricatorRepository::TABLE_PATHCHANGE, + implode(', ', $sql_chunk)); + } + + $this->finishParse(); + } + + private function mercurialPathExists( + PhabricatorRepository $repository, + $path, + $rev) { + + if ($path == '/') { + return true; + } + + // NOTE: For directories, this grabs the entire directory contents, but + // we don't have any more surgical approach available to us in Mercurial. + // We can't use "log" because it doesn't have enough information for us + // to figure out when a directory is deleted by a change. + list($err) = $repository->execLocalCommand( + 'cat --rev %s -- %s > /dev/null', + $rev, + $path); + + if ($err) { + return false; + } else { + return true; + } + } + +} diff --git a/src/applications/repository/worker/commitchangeparser/mercurial/__init__.php b/src/applications/repository/worker/commitchangeparser/mercurial/__init__.php new file mode 100644 index 0000000000..b773603f64 --- /dev/null +++ b/src/applications/repository/worker/commitchangeparser/mercurial/__init__.php @@ -0,0 +1,19 @@ +