Basic support for Mercurial in Diffusion

Summary: Change import script plus almost all the view stuff. Still some rough
edges but this seems to mostly work. Blame is currently unsupported but I think
everything else works properly.

Test Plan:
Imported the hg repository itself. It doesn't immediately seem completely
broken. Here are some screens:

https://secure.phabricator.com/file/view/PHID-FILE-1438b71cc7c4a2eb4569/
https://secure.phabricator.com/file/view/PHID-FILE-3cec4f72f39e7de2d041/
https://secure.phabricator.com/file/view/PHID-FILE-2ea4883f160e8e5098f9/
https://secure.phabricator.com/file/view/PHID-FILE-35f751a36ebf65399ade/

All the parsers were able to churn through it without errors.

Ran the new "reparse.php" script in various one-commit and repository modes.

Browsed/imported some git repos for good measure.

NOTE: The hg repository is only 15,000 commits and around 1,000 files.
Performance is okay but hg doesn't provide performant, native APIs to get some
data efficiently so we have to do some dumb stuff. If some of these interfaces
are cripplingly slow or whatever, let me know and we can start bundling some
Mercurial extensions with Arcanist.

Reviewers: Makinde, jungejason, nh, tuomaspelkonen, aran

Reviewed By: Makinde

CC: aran, Makinde, epriestley

Differential Revision: 960

src/__phutil_library_map__.php

@@ -261,6 +261,13 @@ phutil_register_library_map(array(
     'DiffusionHomeController' => 'applications/diffusion/controller/home',
     'DiffusionLastModifiedController' => 'applications/diffusion/controller/lastmodified',
     'DiffusionLastModifiedQuery' => 'applications/diffusion/query/lastmodified/base',
+    'DiffusionMercurialBranchQuery' => 'applications/diffusion/query/branch/mercurial',
+    'DiffusionMercurialBrowseQuery' => 'applications/diffusion/query/browse/mercurial',
+    'DiffusionMercurialDiffQuery' => 'applications/diffusion/query/diff/mercurial',
+    'DiffusionMercurialFileContentQuery' => 'applications/diffusion/query/filecontent/mercurial',
+    'DiffusionMercurialHistoryQuery' => 'applications/diffusion/query/history/mercurial',
+    'DiffusionMercurialLastModifiedQuery' => 'applications/diffusion/query/lastmodified/mercurial',
+    'DiffusionMercurialRequest' => 'applications/diffusion/request/mercurial',
     'DiffusionPathChange' => 'applications/diffusion/data/pathchange',
     'DiffusionPathChangeQuery' => 'applications/diffusion/query/pathchange/base',
     'DiffusionPathCompleteController' => 'applications/diffusion/controller/pathcomplete',
@@ -582,6 +589,7 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryGitHubNotification' => 'applications/repository/storage/githubnotification',
     'PhabricatorRepositoryGitHubPostReceiveController' => 'applications/repository/controller/github-post-receive',
     'PhabricatorRepositoryListController' => 'applications/repository/controller/list',
+    'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/mercurial',
     'PhabricatorRepositoryMercurialCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/mercurial',
     'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/mercurial',
     'PhabricatorRepositoryMercurialPullDaemon' => 'applications/repository/daemon/mercurialpull',
@@ -927,6 +935,13 @@ phutil_register_library_map(array(
     'DiffusionHistoryTableView' => 'DiffusionView',
     'DiffusionHomeController' => 'DiffusionController',
     'DiffusionLastModifiedController' => 'DiffusionController',
+    'DiffusionMercurialBranchQuery' => 'DiffusionBranchQuery',
+    'DiffusionMercurialBrowseQuery' => 'DiffusionBrowseQuery',
+    'DiffusionMercurialDiffQuery' => 'DiffusionDiffQuery',
+    'DiffusionMercurialFileContentQuery' => 'DiffusionFileContentQuery',
+    'DiffusionMercurialHistoryQuery' => 'DiffusionHistoryQuery',
+    'DiffusionMercurialLastModifiedQuery' => 'DiffusionLastModifiedQuery',
+    'DiffusionMercurialRequest' => 'DiffusionRequest',
     'DiffusionPathCompleteController' => 'DiffusionController',
     'DiffusionPathQueryTestCase' => 'PhabricatorTestCase',
     'DiffusionPathValidateController' => 'DiffusionController',
@@ -1188,10 +1203,8 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryGitHubNotification' => 'PhabricatorRepositoryDAO',
     'PhabricatorRepositoryGitHubPostReceiveController' => 'PhabricatorRepositoryController',
     'PhabricatorRepositoryListController' => 'PhabricatorRepositoryController',
-<<<<<<< HEAD
+    'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
     'PhabricatorRepositoryMercurialCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
-=======
->>>>>>> Add a Mercurial message parser
     'PhabricatorRepositoryMercurialCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
     'PhabricatorRepositoryMercurialPullDaemon' => 'PhabricatorRepositoryPullLocalDaemon',
     'PhabricatorRepositoryPullLocalDaemon' => 'PhabricatorRepositoryDaemon',

src/applications/diffusion/controller/commit/DiffusionCommitController.php

@@ -172,6 +172,7 @@ class DiffusionCommitController extends DiffusionController {
           $vcs_supports_directory_changes = true;
           break;
         case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
+        case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
           $vcs_supports_directory_changes = false;
           break;
         default:

src/applications/diffusion/query/branch/base/DiffusionBranchQuery.php

@@ -33,6 +33,9 @@ abstract class DiffusionBranchQuery {
       case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
         $class = 'DiffusionGitBranchQuery';
         break;
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
+        $class = 'DiffusionMercurialBranchQuery';
+        break;
       default:
         throw new Exception("Unsupported VCS!");
     }

src/applications/diffusion/query/branch/mercurial/DiffusionMercurialBranchQuery.php

@@ -0,0 +1,40 @@
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+final class DiffusionMercurialBranchQuery extends DiffusionBranchQuery {
+
+  protected function executeQuery() {
+    $drequest = $this->getRequest();
+    $repository = $drequest->getRepository();
+
+    list($stdout) = $repository->execxLocalCommand(
+      'branches');
+    $branch_info = ArcanistMercurialParser::parseMercurialBranches($stdout);
+
+    $branches = array();
+    foreach ($branch_info as $name => $info) {
+      $branch = new DiffusionBranchInformation();
+      $branch->setName($name);
+      $branch->setHeadCommitIdentifier($info['rev']);
+      $branches[] = $branch;
+    }
+
+    return $branches;
+  }
+
+}

src/applications/diffusion/query/branch/mercurial/__init__.php

@@ -0,0 +1,15 @@
+<?php
+/**
+ * This file is automatically generated. Lint this module to rebuild it.
+ * @generated
+ */
+
+
+
+phutil_require_module('arcanist', 'repository/parser/mercurial');
+
+phutil_require_module('phabricator', 'applications/diffusion/data/branch');
+phutil_require_module('phabricator', 'applications/diffusion/query/branch/base');
+
+
+phutil_require_source('DiffusionMercurialBranchQuery.php');

src/applications/diffusion/query/browse/base/DiffusionBrowseQuery.php

@@ -46,6 +46,9 @@ abstract class DiffusionBrowseQuery {
         // TODO: Verify local-path?
         $class = 'DiffusionGitBrowseQuery';
         break;
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
+        $class = 'DiffusionMercurialBrowseQuery';
+        break;
       case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
         $class = 'DiffusionSvnBrowseQuery';
         break;

src/applications/diffusion/query/browse/mercurial/DiffusionMercurialBrowseQuery.php

@@ -0,0 +1,95 @@
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+final class DiffusionMercurialBrowseQuery extends DiffusionBrowseQuery {
+
+  protected function executeQuery() {
+    $drequest = $this->getRequest();
+    $repository = $drequest->getRepository();
+
+    $path = $drequest->getPath();
+    $commit = $drequest->getCommit();
+
+    // TODO: This is a really really awful mess but Mercurial doesn't offer
+    // an equivalent of "git ls-files -- directory". If it's any comfort, this
+    // is what "hgweb" does too, see:
+    //
+    //   http://selenic.com/repo/hg/file/91dc8878f888/mercurial/hgweb/webcommands.py#l320
+    //
+    // derp derp derp derp
+    //
+    // Anyway, figure out what's in this path by applying massive amounts
+    // of brute force.
+
+    list($entire_manifest) = $repository->execxLocalCommand(
+      'manifest --rev %s',
+      $commit);
+    $entire_manifest = explode("\n", $entire_manifest);
+
+    $results = array();
+
+    $match_against = trim($path, '/');
+    $match_len = strlen($match_against);
+
+    // For the root, don't trim. For other paths, trim the "/" after we match.
+    // We need this because Mercurial's canonical paths have no leading "/",
+    // but ours do.
+    $trim_len = $match_len ? $match_len + 1 : 0;
+
+    foreach ($entire_manifest as $path) {
+      if (strncmp($path, $match_against, $match_len)) {
+        continue;
+      }
+      if (!strlen($path)) {
+        continue;
+      }
+      $remainder = substr($path, $trim_len);
+      if (!strlen($remainder)) {
+        // There is a file with this exact name in the manifest, so clearly
+        // it's a file.
+        $this->reason = self::REASON_IS_FILE;
+        return array();
+      }
+      $parts = explode('/', $remainder);
+      if (count($parts) == 1) {
+        $type = DifferentialChangeType::FILE_NORMAL;
+      } else {
+        $type = DifferentialChangeType::FILE_DIRECTORY;
+      }
+      $results[reset($parts)] = $type;
+    }
+
+    foreach ($results as $key => $type) {
+      $result = new DiffusionRepositoryPath();
+      $result->setPath($key);
+      $result->setFileType($type);
+
+      $results[$key] = $result;
+    }
+
+    if (empty($results)) {
+      // TODO: Detect "deleted" by issuing "hg log"?
+
+      $this->reason = self::REASON_IS_NONEXISTENT;
+    }
+
+
+    return $results;
+  }
+
+}

src/applications/diffusion/query/browse/mercurial/__init__.php

@@ -0,0 +1,14 @@
+<?php
+/**
+ * This file is automatically generated. Lint this module to rebuild it.
+ * @generated
+ */
+
+
+
+phutil_require_module('phabricator', 'applications/differential/constants/changetype');
+phutil_require_module('phabricator', 'applications/diffusion/data/repositorypath');
+phutil_require_module('phabricator', 'applications/diffusion/query/browse/base');
+
+
+phutil_require_source('DiffusionMercurialBrowseQuery.php');

src/applications/diffusion/query/diff/base/DiffusionDiffQuery.php

@@ -34,6 +34,9 @@ abstract class DiffusionDiffQuery {
       case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
         $class = 'DiffusionGitDiffQuery';
         break;
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
+        $class = 'DiffusionMercurialDiffQuery';
+        break;
       case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
         $class = 'DiffusionSvnDiffQuery';
         break;

src/applications/diffusion/query/diff/mercurial/DiffusionMercurialDiffQuery.php

@@ -0,0 +1,56 @@
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+final class DiffusionMercurialDiffQuery extends DiffusionDiffQuery {
+
+  protected function executeQuery() {
+    $drequest = $this->getRequest();
+    $repository = $drequest->getRepository();
+
+    $effective_commit = $this->getEffectiveCommit();
+    if (!$effective_commit) {
+      return null;
+    }
+    // TODO: This side effect is kind of skethcy.
+    $drequest->setCommit($effective_commit);
+
+    $path = $drequest->getPath();
+
+    list($raw_diff) = $repository->execxLocalCommand(
+      'diff -U %d --git --change %s -- %s',
+      65535,
+      $effective_commit,
+      $path);
+
+    $parser = new ArcanistDiffParser();
+    $parser->setDetectBinaryFiles(true);
+    $changes = $parser->parseDiff($raw_diff);
+
+    $diff = DifferentialDiff::newFromRawChanges($changes);
+    $changesets = $diff->getChangesets();
+    $changeset = reset($changesets);
+
+    $this->renderingReference =
+      $drequest->getBranchURIComponent($drequest->getBranch()).
+      $drequest->getPath().';'.
+      $drequest->getCommit();
+
+    return $changeset;
+  }
+
+}

src/applications/diffusion/query/diff/mercurial/__init__.php

@@ -0,0 +1,15 @@
+<?php
+/**
+ * This file is automatically generated. Lint this module to rebuild it.
+ * @generated
+ */
+
+
+
+phutil_require_module('arcanist', 'parser/diff');
+
+phutil_require_module('phabricator', 'applications/differential/storage/diff');
+phutil_require_module('phabricator', 'applications/diffusion/query/diff/base');
+
+
+phutil_require_source('DiffusionMercurialDiffQuery.php');

src/applications/diffusion/query/filecontent/base/DiffusionFileContentQuery.php

@@ -35,6 +35,9 @@ abstract class DiffusionFileContentQuery {
       case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
         $class = 'DiffusionGitFileContentQuery';
         break;
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
+        $class = 'DiffusionMercurialFileContentQuery';
+        break;
       case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
         $class = 'DiffusionSvnFileContentQuery';
         break;

src/applications/diffusion/query/filecontent/mercurial/DiffusionMercurialFileContentQuery.php

@@ -0,0 +1,47 @@
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+final class DiffusionMercurialFileContentQuery
+  extends DiffusionFileContentQuery {
+
+  protected function executeQuery() {
+    $drequest = $this->getRequest();
+
+    $repository = $drequest->getRepository();
+    $path = $drequest->getPath();
+    $commit = $drequest->getCommit();
+
+    list($corpus) = $repository->execxLocalCommand(
+      'cat --rev %s -- %s',
+      $commit,
+      $path);
+
+    $file_content = new DiffusionFileContent();
+    $file_content->setCorpus($corpus);
+
+    return $file_content;
+  }
+
+
+  protected function tokenizeLine($line) {
+    // TODO: Support blame.
+    throw new Exception(
+      "Diffusion does not currently support blame for Mercurial.");
+  }
+
+}

src/applications/diffusion/query/filecontent/mercurial/__init__.php

@@ -0,0 +1,13 @@
+<?php
+/**
+ * This file is automatically generated. Lint this module to rebuild it.
+ * @generated
+ */
+
+
+
+phutil_require_module('phabricator', 'applications/diffusion/data/filecontent');
+phutil_require_module('phabricator', 'applications/diffusion/query/filecontent/base');
+
+
+phutil_require_source('DiffusionMercurialFileContentQuery.php');

src/applications/diffusion/query/history/base/DiffusionHistoryQuery.php

@@ -41,6 +41,9 @@ abstract class DiffusionHistoryQuery {
       case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
         $class = 'DiffusionSvnHistoryQuery';
         break;
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
+        $class = 'DiffusionMercurialHistoryQuery';
+        break;
       default:
         throw new Exception("Unsupported VCS!");
     }
@@ -90,4 +93,76 @@ abstract class DiffusionHistoryQuery {
   }
 
   abstract protected function executeQuery();
+
+  final protected function loadHistoryForCommitIdentifiers(array $identifiers) {
+    if (!$identifiers) {
+      return array();
+    }
+
+    $commits = array();
+    $commit_data = array();
+    $path_changes = array();
+
+    $drequest = $this->getRequest();
+    $repository = $drequest->getRepository();
+
+    $path = $drequest->getPath();
+
+    $commits = id(new PhabricatorRepositoryCommit())->loadAllWhere(
+      'repositoryID = %d AND commitIdentifier IN (%Ls)',
+        $repository->getID(),
+      $identifiers);
+    $commits = mpull($commits, null, 'getCommitIdentifier');
+
+    if (!$commits) {
+      return array();
+    }
+
+    $commit_data = id(new PhabricatorRepositoryCommitData())->loadAllWhere(
+      'commitID in (%Ld)',
+      mpull($commits, 'getID'));
+    $commit_data = mpull($commit_data, null, 'getCommitID');
+
+    $conn_r = $repository->establishConnection('r');
+
+    $path_normal = DiffusionPathIDQuery::normalizePath($path);
+    $paths = queryfx_all(
+      $conn_r,
+      'SELECT id, path FROM %T WHERE path IN (%Ls)',
+      PhabricatorRepository::TABLE_PATH,
+      array($path_normal));
+    $paths = ipull($paths, 'id', 'path');
+    $path_id = idx($paths, $path_normal);
+
+    $path_changes = queryfx_all(
+      $conn_r,
+      'SELECT * FROM %T WHERE commitID IN (%Ld) AND pathID = %d',
+      PhabricatorRepository::TABLE_PATHCHANGE,
+      mpull($commits, 'getID'),
+      $path_id);
+    $path_changes = ipull($path_changes, null, 'commitID');
+
+    $history = array();
+    foreach ($identifiers as $identifier) {
+      $item = new DiffusionPathChange();
+      $item->setCommitIdentifier($identifier);
+      $commit = idx($commits, $identifier);
+      if ($commit) {
+        $item->setCommit($commit);
+        $data = idx($commit_data, $commit->getID());
+        if ($data) {
+          $item->setCommitData($data);
+        }
+        $change = idx($path_changes, $commit->getID());
+        if ($change) {
+          $item->setChangeType($change['changeType']);
+          $item->setFileType($change['fileType']);
+        }
+      }
+      $history[] = $item;
+    }
+
+    return $history;
+  }
+
 }

src/applications/diffusion/query/history/base/__init__.php

@@ -6,9 +6,16 @@
 
 
 
+phutil_require_module('phabricator', 'applications/diffusion/data/pathchange');
+phutil_require_module('phabricator', 'applications/diffusion/query/pathid/base');
 phutil_require_module('phabricator', 'applications/repository/constants/repositorytype');
+phutil_require_module('phabricator', 'applications/repository/storage/commit');
+phutil_require_module('phabricator', 'applications/repository/storage/commitdata');
+phutil_require_module('phabricator', 'applications/repository/storage/repository');
+phutil_require_module('phabricator', 'storage/queryfx');
 
 phutil_require_module('phutil', 'symbols');
+phutil_require_module('phutil', 'utils');
 
 
 phutil_require_source('DiffusionHistoryQuery.php');

src/applications/diffusion/query/history/git/DiffusionGitHistoryQuery.php

@@ -43,67 +43,7 @@ final class DiffusionGitHistoryQuery extends DiffusionHistoryQuery {
     $hashes = explode("\n", $stdout);
     $hashes = array_filter($hashes);
 
-    $commits = array();
+    return $this->loadHistoryForCommitIdentifiers($hashes);
-    $commit_data = array();
-    $path_changes = array();
-
-    $conn_r = $repository->establishConnection('r');
-
-    if ($hashes) {
-      $commits = id(new PhabricatorRepositoryCommit())->loadAllWhere(
-        'repositoryID = %d AND commitIdentifier IN (%Ls)',
-          $repository->getID(),
-        $hashes);
-      $commits = mpull($commits, null, 'getCommitIdentifier');
-      if ($commits) {
-        $commit_data = id(new PhabricatorRepositoryCommitData())->loadAllWhere(
-          'commitID in (%Ld)',
-          mpull($commits, 'getID'));
-        $commit_data = mpull($commit_data, null, 'getCommitID');
-      }
-
-      if ($commits) {
-        $path_normal = '/'.trim($path, '/');
-        $paths = queryfx_all(
-          $conn_r,
-          'SELECT id, path FROM %T WHERE path IN (%Ls)',
-          PhabricatorRepository::TABLE_PATH,
-          array($path_normal));
-        $paths = ipull($paths, 'id', 'path');
-        $path_id = idx($paths, $path_normal);
-
-        $path_changes = queryfx_all(
-          $conn_r,
-          'SELECT * FROM %T WHERE commitID IN (%Ld) AND pathID = %d',
-          PhabricatorRepository::TABLE_PATHCHANGE,
-          mpull($commits, 'getID'),
-          $path_id);
-        $path_changes = ipull($path_changes, null, 'commitID');
-      }
-    }
-
-
-    $history = array();
-    foreach ($hashes as $hash) {
-      $item = new DiffusionPathChange();
-      $item->setCommitIdentifier($hash);
-      $commit = idx($commits, $hash);
-      if ($commit) {
-        $item->setCommit($commit);
-        $data = idx($commit_data, $commit->getID());
-        if ($data) {
-          $item->setCommitData($data);
-        }
-        $change = idx($path_changes, $commit->getID());
-        if ($change) {
-          $item->setChangeType($change['changeType']);
-          $item->setFileType($change['fileType']);
-        }
-      }
-      $history[] = $item;
-    }
-
-    return $history;
   }
 
 }

src/applications/diffusion/query/history/git/__init__.php

@@ -6,15 +6,9 @@
 
 
 
-phutil_require_module('phabricator', 'applications/diffusion/data/pathchange');
 phutil_require_module('phabricator', 'applications/diffusion/query/history/base');
-phutil_require_module('phabricator', 'applications/repository/storage/commit');
-phutil_require_module('phabricator', 'applications/repository/storage/commitdata');
-phutil_require_module('phabricator', 'applications/repository/storage/repository');
-phutil_require_module('phabricator', 'storage/queryfx');
 
 phutil_require_module('phutil', 'future/exec');
-phutil_require_module('phutil', 'utils');
 
 
 phutil_require_source('DiffusionGitHistoryQuery.php');

src/applications/diffusion/query/history/mercurial/DiffusionMercurialHistoryQuery.php

@@ -0,0 +1,45 @@
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+final class DiffusionMercurialHistoryQuery extends DiffusionHistoryQuery {
+
+  protected function executeQuery() {
+    $drequest = $this->getRequest();
+
+    $repository = $drequest->getRepository();
+    $path = $drequest->getPath();
+    $commit_hash = $drequest->getCommit();
+
+    $path = DiffusionPathIDQuery::normalizePath($path);
+
+    list($stdout) = $repository->execxLocalCommand(
+      'log --template %s --limit %d --branch %s --rev %s:0 -- %s',
+      '{node}\\n',
+      ($this->getOffset() + $this->getLimit()), // No '--skip' in Mercurial.
+      $drequest->getBranch(),
+      $commit_hash,
+      nonempty(ltrim($path, '/'), '.'));
+
+    $hashes = explode("\n", $stdout);
+    $hashes = array_filter($hashes);
+    $hashes = array_slice($hashes, $this->getOffset());
+
+    return $this->loadHistoryForCommitIdentifiers($hashes);
+  }
+
+}

src/applications/diffusion/query/history/mercurial/__init__.php

@@ -0,0 +1,15 @@
+<?php
+/**
+ * This file is automatically generated. Lint this module to rebuild it.
+ * @generated
+ */
+
+
+
+phutil_require_module('phabricator', 'applications/diffusion/query/history/base');
+phutil_require_module('phabricator', 'applications/diffusion/query/pathid/base');
+
+phutil_require_module('phutil', 'utils');
+
+
+phutil_require_source('DiffusionMercurialHistoryQuery.php');

src/applications/diffusion/query/lastmodified/base/DiffusionLastModifiedQuery.php

@@ -33,6 +33,9 @@ abstract class DiffusionLastModifiedQuery {
       case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
         $class = 'DiffusionGitLastModifiedQuery';
         break;
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
+        $class = 'DiffusionMercurialLastModifiedQuery';
+        break;
       case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
         $class = 'DiffusionSvnLastModifiedQuery';
         break;

src/applications/diffusion/query/lastmodified/mercurial/DiffusionMercurialLastModifiedQuery.php

@@ -0,0 +1,50 @@
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+final class DiffusionMercurialLastModifiedQuery
+  extends DiffusionLastModifiedQuery {
+
+  protected function executeQuery() {
+    $drequest = $this->getRequest();
+    $repository = $drequest->getRepository();
+
+    $path = $drequest->getPath();
+
+    // TODO: Share some of this with History query.
+    list($hash) = $repository->execxLocalCommand(
+      'log --template %s --limit 1 --branch %s --rev %s:0 -- %s',
+      '{node}\\n',
+      $drequest->getBranch(),
+      $drequest->getCommit(),
+      nonempty(ltrim($path, '/'), '.'));
+    $hash = trim($hash);
+
+    $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
+      'repositoryID = %d AND commitIdentifier = %s',
+      $repository->getID(),
+      $hash);
+    if ($commit) {
+      $commit_data = id(new PhabricatorRepositoryCommitData())->loadOneWhere(
+        'commitID = %d',
+        $commit->getID());
+    }
+
+    return array($commit, $commit_data);
+  }
+
+}

src/applications/diffusion/query/lastmodified/mercurial/__init__.php

@@ -0,0 +1,16 @@
+<?php
+/**
+ * This file is automatically generated. Lint this module to rebuild it.
+ * @generated
+ */
+
+
+
+phutil_require_module('phabricator', 'applications/diffusion/query/lastmodified/base');
+phutil_require_module('phabricator', 'applications/repository/storage/commit');
+phutil_require_module('phabricator', 'applications/repository/storage/commitdata');
+
+phutil_require_module('phutil', 'utils');
+
+
+phutil_require_source('DiffusionMercurialLastModifiedQuery.php');

src/applications/diffusion/request/base/DiffusionRequest.php

@@ -55,6 +55,9 @@ class DiffusionRequest {
       case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
         $class = 'DiffusionSvnRequest';
         break;
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
+        $class = 'DiffusionMercurialRequest';
+        break;
       default:
         $class = 'DiffusionRequest';
         break;

src/applications/diffusion/request/mercurial/DiffusionMercurialRequest.php

@@ -0,0 +1,81 @@
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// TODO: This has some minor code duplication vs the Git request that could be
+// shared.
+class DiffusionMercurialRequest extends DiffusionRequest {
+
+  protected function initializeFromAphrontRequestDictionary(array $data) {
+    parent::initializeFromAphrontRequestDictionary($data);
+
+    $path = $this->path;
+    $parts = explode('/', $path);
+
+    $branch = array_shift($parts);
+    if ($branch != ':') {
+      $this->branch = $this->decodeBranchName($branch);
+    }
+
+    foreach ($parts as $key => $part) {
+      if ($part == '..') {
+        unset($parts[$key]);
+      }
+    }
+
+    $this->path = implode('/', $parts);
+  }
+
+  public function getBranch() {
+    if ($this->branch) {
+      return $this->branch;
+    }
+    if ($this->repository) {
+      return $this->repository->getDetail('default-branch', 'default');
+    }
+    throw new Exception("Unable to determine branch!");
+  }
+
+  public function getUriPath() {
+    return '/diffusion/'.$this->getCallsign().'/browse/'.
+      $this->getBranchURIComponent($this->branch).$this->path;
+  }
+
+  public function getCommit() {
+    if ($this->commit) {
+      return $this->commit;
+    }
+    return $this->getBranch();
+  }
+
+  public function getStableCommitName() {
+    return substr($this->stableCommitName, 0, 16);
+  }
+
+  public function getBranchURIComponent($branch) {
+    return $this->encodeBranchName($branch).'/';
+  }
+
+  private function decodeBranchName($branch) {
+    return str_replace(':', '/', $branch);
+  }
+
+  private function encodeBranchName($branch) {
+    return str_replace('/', ':', $branch);
+  }
+
+}

src/applications/diffusion/request/mercurial/__init__.php

@@ -0,0 +1,12 @@
+<?php
+/**
+ * This file is automatically generated. Lint this module to rebuild it.
+ * @generated
+ */
+
+
+
+phutil_require_module('phabricator', 'applications/diffusion/request/base');
+
+
+phutil_require_source('DiffusionMercurialRequest.php');

src/applications/repository/worker/commitchangeparser/mercurial/PhabricatorRepositoryMercurialCommitChangeParserWorker.php

@@ -0,0 +1,354 @@
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+class PhabricatorRepositoryMercurialCommitChangeParserWorker
+  extends PhabricatorRepositoryCommitChangeParserWorker {
+
+  protected function parseCommit(
+    PhabricatorRepository $repository,
+    PhabricatorRepositoryCommit $commit) {
+
+    $full_name = 'r'.$repository->getCallsign().$commit->getCommitIdentifier();
+    echo "Parsing {$full_name}...\n";
+    if ($this->isBadCommit($full_name)) {
+      echo "This commit is marked bad!\n";
+      return;
+    }
+
+    list($stdout) = $repository->execxLocalCommand(
+      'status -C --change %s',
+      $commit->getCommitIdentifier());
+    $status = ArcanistMercurialParser::parseMercurialStatusDetails($stdout);
+
+    $common_attributes = array(
+      'repositoryID'    => $repository->getID(),
+      'commitID'        => $commit->getID(),
+      'commitSequence'  => $commit->getEpoch(),
+    );
+
+    $changes = array();
+
+    // Like Git, Mercurial doesn't track directories directly. We need to infer
+    // directory creation and removal by observing file creation and removal
+    // and testing if the directories in question are previously empty (thus,
+    // created) or subsequently empty (thus, removed).
+    $maybe_new_directories = array();
+    $maybe_del_directories = array();
+    $all_directories = array();
+
+    // Parse the basic information from "hg status", which shows files that
+    // were directly affected by the change.
+    foreach ($status as $path => $path_info) {
+      $path = '/'.$path;
+      $flags = $path_info['flags'];
+      $change_target = $path_info['from'] ? '/'.$path_info['from'] : null;
+
+      $changes[$path] = array(
+        'path'            => $path,
+        'isDirect'        => true,
+
+        'targetPath'      => $change_target,
+        'targetCommitID'  => $change_target ? $commit->getID() : null,
+
+        // We're going to fill these in shortly.
+        'changeType'      => null,
+        'fileType'        => null,
+
+        'flags'           => $flags,
+      ) + $common_attributes;
+
+      if ($flags & ArcanistRepositoryAPI::FLAG_ADDED) {
+        $maybe_new_directories[] = dirname($path);
+      } else if ($flags & ArcanistRepositoryAPI::FLAG_DELETED) {
+        $maybe_del_directories[] = dirname($path);
+      }
+      $all_directories[] = dirname($path);
+    }
+
+    // Add change information for each source path which doesn't appear in the
+    // status. These files were copied, but were not modified. We also know they
+    // must exist.
+    foreach ($changes as $path => $change) {
+      $from = $change['targetPath'];
+      if ($from && empty($changes[$from])) {
+        $changes[$from] = array(
+          'path'            => $from,
+          'isDirect'        => false,
+
+          'targetPath'      => null,
+          'targetCommitID'  => null,
+
+          'changeType'      => DifferentialChangeType::TYPE_COPY_AWAY,
+          'fileType'        => null,
+
+          'flags'           => 0,
+        ) + $common_attributes;
+      }
+    }
+
+    $away = array();
+    foreach ($changes as $path => $change) {
+      if ($path['targetPath']) {
+        $away[$path['targetPath']][] = $path;
+      }
+    }
+
+    // Now that we have all the direct changes, figure out change types.
+    foreach ($changes as $path => $change) {
+      $flags = $change['flags'];
+      $from = $change['targetPath'];
+      if ($from) {
+        $target = $changes[$from];
+      } else {
+        $target = null;
+      }
+
+      if ($flags & ArcanistRepositoryAPI::FLAG_ADDED) {
+        if ($target) {
+          if ($target['flags'] & ArcanistRepositoryAPI::FLAG_DELETED) {
+            $change_type = DifferentialChangeType::TYPE_MOVE_HERE;
+          } else {
+            $change_type = DifferentialChangeType::TYPE_COPY_HERE;
+          }
+        } else {
+          $change_type = DifferentialChangeType::TYPE_ADD;
+        }
+      } else if ($flags & ArcanistRepositoryAPI::FLAG_DELETED) {
+        if (isset($away[$path])) {
+          if (count($away[$path]) > 1) {
+            $change_type = DifferentialChangeType::TYPE_MULTICOPY;
+          } else {
+            $change_type = DifferentialChangeType::TYPE_MOVE_AWAY;
+          }
+        } else {
+          $change_type = DifferentialChangeType::TYPE_DELETE;
+        }
+      } else {
+        if (isset($away[$path])) {
+          $change_type = DifferentialChangeType::TYPE_COPY_AWAY;
+        } else {
+          $change_type = DifferentialChangeType::TYPE_CHANGE;
+        }
+      }
+
+      $changes[$path]['changeType'] = $change_type;
+    }
+
+    // Go through all the affected directories and identify any which were
+    // actually added or deleted.
+    $dir_status = array();
+
+    foreach ($maybe_del_directories as $dir) {
+      $exists = false;
+      foreach (DiffusionPathIDQuery::expandPathToRoot($dir) as $path) {
+        if (isset($dir_status[$path])) {
+          break;
+        }
+
+        // If we know some child exists, we know this path exists. If we don't
+        // know that a child exists, test if this directory still exists.
+        if (!$exists) {
+          $exists = $this->mercurialPathExists(
+            $repository,
+            $path,
+            $commit->getCommitIdentifier());
+        }
+
+        if ($exists) {
+          $dir_status[$path] = DifferentialChangeType::TYPE_CHILD;
+        } else {
+          $dir_status[$path] = DifferentialChangeType::TYPE_DELETE;
+        }
+      }
+    }
+
+    list($stdout) = $repository->execxLocalCommand(
+      'parents --rev %s --style default',
+      $commit->getCommitIdentifier());
+    $parents = ArcanistMercurialParser::parseMercurialLog($stdout);
+    $parent = reset($parents);
+    if ($parent) {
+      // TODO: We should expand this to a full 40-character hash using "hg id".
+      $parent = $parent['rev'];
+    }
+    foreach ($maybe_new_directories as $dir) {
+      $exists = false;
+      foreach (DiffusionPathIDQuery::expandPathToRoot($dir) as $path) {
+        if (isset($dir_status[$path])) {
+          break;
+        }
+        if (!$exists) {
+          if ($parent) {
+            $exists = $this->mercurialPathExists($repository, $path, $parent);
+          } else {
+            $exists = false;
+          }
+        }
+        if ($exists) {
+          $dir_status[$path] = DifferentialChangeType::TYPE_CHILD;
+        } else {
+          $dir_status[$path] = DifferentialChangeType::TYPE_ADD;
+        }
+      }
+    }
+
+    foreach ($all_directories as $dir) {
+      foreach (DiffusionPathIDQuery::expandPathToRoot($dir) as $path) {
+        if (isset($dir_status[$path])) {
+          break;
+        }
+        $dir_status[$path] = DifferentialChangeType::TYPE_CHILD;
+      }
+    }
+
+    // Merge all the directory statuses into the path statuses.
+    foreach ($dir_status as $path => $status) {
+      if (isset($changes[$path])) {
+        // TODO: The UI probably doesn't handle any of these cases with
+        // terrible elegance, but they are exceedingly rare.
+
+        $existing_type = $changes[$path]['changeType'];
+        if ($existing_type == DifferentialChangeType::TYPE_DELETE) {
+          // This change removes a file, replaces it with a directory, and then
+          // adds children of that directory. Mark it as a "change" instead,
+          // and make the type a directory.
+          $changes[$path]['fileType'] = DifferentialChangeType::FILE_DIRECTORY;
+          $changes[$path]['changeType'] = DifferentialChangeType::TYPE_CHANGE;
+        } else if ($existing_type == DifferentialChangeType::TYPE_MOVE_AWAY ||
+                   $existing_type == DifferentialChangeType::TYPE_MULTICOPY) {
+          // This change moves or copies a file, replaces it with a directory,
+          // and then adds children to that directory. Mark it as "copy away"
+          // instead of whatever it was, and make the type a directory.
+          $changes[$path]['fileType'] = DifferentialChangeType::FILE_DIRECTORY;
+          $changes[$path]['changeType']
+            = DifferentialChangeType::TYPE_COPY_AWAY;
+        } else if ($existing_type == DifferentialChangeType::TYPE_ADD) {
+          // This change removes a diretory and replaces it with a file. Mark
+          // it as "change" instead of "add".
+          $changes[$path]['changeType'] = DifferentialChangeType::TYPE_CHANGE;
+        }
+
+        continue;
+      }
+
+      $changes[$path] = array(
+        'path' => $path,
+        'isDirect' => ($status == DifferentialChangeType::TYPE_CHILD)
+          ? false
+          : true,
+        'fileType' => DifferentialChangeType::FILE_DIRECTORY,
+        'changeType' => $status,
+
+        'targetPath'      => null,
+        'targetCommitID'  => null,
+      ) + $common_attributes;
+    }
+
+    // TODO: use "hg diff --git" to figure out which files are symlinks.
+    foreach ($changes as $path => $change) {
+      if (empty($change['fileType'])) {
+        $changes[$path]['fileType'] = DifferentialChangeType::FILE_NORMAL;
+      }
+    }
+
+    $all_paths = array();
+    foreach ($changes as $path => $change) {
+      $all_paths[$path] = true;
+      if ($change['targetPath']) {
+        $all_paths[$change['targetPath']] = true;
+      }
+    }
+
+    $path_map = $this->lookupOrCreatePaths(array_keys($all_paths));
+
+    foreach ($changes as $key => $change) {
+      $changes[$key]['pathID'] = $path_map[$change['path']];
+      if ($change['targetPath']) {
+        $changes[$key]['targetPathID'] = $path_map[$change['targetPath']];
+      } else {
+        $changes[$key]['targetPathID'] = null;
+      }
+    }
+
+    $conn_w = $repository->establishConnection('w');
+
+    $changes_sql = array();
+    foreach ($changes as $change) {
+      $values = array(
+        (int)$change['repositoryID'],
+        (int)$change['pathID'],
+        (int)$change['commitID'],
+        $change['targetPathID']
+          ? (int)$change['targetPathID']
+          : 'null',
+        $change['targetCommitID']
+          ? (int)$change['targetCommitID']
+          : 'null',
+        (int)$change['changeType'],
+        (int)$change['fileType'],
+        (int)$change['isDirect'],
+        (int)$change['commitSequence'],
+      );
+      $changes_sql[] = '('.implode(', ', $values).')';
+    }
+
+    queryfx(
+      $conn_w,
+      'DELETE FROM %T WHERE commitID = %d',
+      PhabricatorRepository::TABLE_PATHCHANGE,
+      $commit->getID());
+    foreach (array_chunk($changes_sql, 256) as $sql_chunk) {
+      queryfx(
+        $conn_w,
+        'INSERT INTO %T
+          (repositoryID, pathID, commitID, targetPathID, targetCommitID,
+            changeType, fileType, isDirect, commitSequence)
+          VALUES %Q',
+        PhabricatorRepository::TABLE_PATHCHANGE,
+        implode(', ', $sql_chunk));
+    }
+
+    $this->finishParse();
+  }
+
+  private function mercurialPathExists(
+    PhabricatorRepository $repository,
+    $path,
+    $rev) {
+
+    if ($path == '/') {
+      return true;
+    }
+
+    // NOTE: For directories, this grabs the entire directory contents, but
+    // we don't have any more surgical approach available to us in Mercurial.
+    // We can't use "log" because it doesn't have enough information for us
+    // to figure out when a directory is deleted by a change.
+    list($err) = $repository->execLocalCommand(
+      'cat --rev %s -- %s > /dev/null',
+      $rev,
+      $path);
+
+    if ($err) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+}

src/applications/repository/worker/commitchangeparser/mercurial/__init__.php

@@ -0,0 +1,19 @@
+<?php
+/**
+ * This file is automatically generated. Lint this module to rebuild it.
+ * @generated
+ */
+
+
+
+phutil_require_module('arcanist', 'repository/api/base');
+phutil_require_module('arcanist', 'repository/parser/mercurial');
+
+phutil_require_module('phabricator', 'applications/differential/constants/changetype');
+phutil_require_module('phabricator', 'applications/diffusion/query/pathid/base');
+phutil_require_module('phabricator', 'applications/repository/storage/repository');
+phutil_require_module('phabricator', 'applications/repository/worker/commitchangeparser/base');
+phutil_require_module('phabricator', 'storage/queryfx');
+
+
+phutil_require_source('PhabricatorRepositoryMercurialCommitChangeParserWorker.php');

src/applications/repository/worker/commitchangeparser/svn/PhabricatorRepositorySvnCommitChangeParserWorker.php

@@ -758,6 +758,7 @@ class PhabricatorRepositorySvnCommitChangeParserWorker
     return $map;
   }
 
+  // TODO: Replace with DiffusionPathIDQuery::getParentPath().
   private function getParentPath($path) {
     $path = rtrim($path, '/');
     $path = dirname($path);
@@ -767,6 +768,7 @@ class PhabricatorRepositorySvnCommitChangeParserWorker
     return $path;
   }
 
+  // TODO: Replace with DiffusionPathIDQuery::expandPathToRoot().
   private function expandAllParentPaths($path, $include_self = false) {
     $parents = array();
     if ($include_self) {