Improve parser scalability, fix a bug or two, provide 'phd', the Phabricator

Daemon interface.

bin/phd

@@ -0,0 +1 @@
+../scripts/daemon/phabricator_daemon_launcher.php

scripts/__init_script__.php

@@ -25,8 +25,10 @@ if (!@constant('__LIBPHUTIL__')) {
   exit(1);
 }
 
-if (!ini_get('date.timezone')) {
-  date_default_timezone_set('America/Los_Angeles');
-}
-
 phutil_load_library(dirname(__FILE__).'/../src/');
+
+phutil_require_module('phutil', 'symbols');
+
+function __autoload($class) {
+  PhutilSymbolLoader::loadClass($class);
+}

scripts/daemon/phabricator_daemon_launcher.php

@@ -0,0 +1,81 @@
+#!/usr/bin/env php
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$root = dirname(dirname(dirname(__FILE__)));
+require_once $root.'/scripts/__init_script__.php';
+require_once $root.'/scripts/__init_env__.php';
+
+
+
+switch (isset($argv[1]) ? $argv[1] : 'help') {
+  case 'parse-commit':
+    $commit = isset($argv[2]) ? $argv[2] : null;
+    if (!$commit) {
+      throw new Exception("Provide a commit to parse!");
+    }
+    $matches = null;
+    if (!preg_match('/r([A-Z]+)([a-z0-9]+)/', $commit, $matches)) {
+      throw new Exception("Can't parse commit identifier!");
+    }
+    $repo = id(new PhabricatorRepository())->loadOneWhere(
+      'callsign = %s',
+      $matches[1]);
+    if (!$repo) {
+      throw new Exception("Unknown repository!");
+    }
+    $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
+      'repositoryID = %d AND commitIdentifier = %s',
+      $repo->getID(),
+      $matches[2]);
+    if (!$commit) {
+      throw new Exception('Unknown commit.');
+    }
+
+    switch ($repo->getVersionControlSystem()) {
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
+        $worker = new PhabricatorRepositoryGitCommitChangeParserWorker(
+          $commit->getID());
+        break;
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
+        $worker = new PhabricatorRepositorySvnCommitChangeParserWorker(
+          $commit->getID());
+        break;
+      default:
+        throw new Exception("Unknown repository type!");
+    }
+
+    ExecFuture::pushEchoMode(true);
+
+    $worker->doWork();
+
+    echo "Done.\n";
+
+    break;
+  case '--help':
+  case 'help':
+  default:
+    echo <<<EOHELP
+phd - phabricator daemon launcher
+
+parse-commit <rXnnnn>
+  Parse a single commit.
+
+EOHELP;
+    exit(1);
+}

src/applications/diffusion/controller/home/__init__.php

@@ -7,12 +7,12 @@
 
 
 phutil_require_module('phabricator', 'applications/diffusion/controller/base');
+phutil_require_module('phabricator', 'applications/diffusion/view/base');
 phutil_require_module('phabricator', 'applications/repository/storage/commit');
 phutil_require_module('phabricator', 'applications/repository/storage/repository');
 phutil_require_module('phabricator', 'storage/queryfx');
 phutil_require_module('phabricator', 'view/control/table');
 phutil_require_module('phabricator', 'view/layout/panel');
-phutil_require_module('phabricator', 'view/utils');
 
 phutil_require_module('phutil', 'markup');
 phutil_require_module('phutil', 'utils');

src/applications/diffusion/controller/repository/__init__.php

@@ -15,5 +15,7 @@ phutil_require_module('phabricator', 'applications/diffusion/view/browsetable');
 phutil_require_module('phabricator', 'applications/diffusion/view/historytable');
 phutil_require_module('phabricator', 'view/layout/panel');
 
+phutil_require_module('phutil', 'markup');
+
 
 phutil_require_source('DiffusionRepositoryController.php');

src/applications/diffusion/query/history/svn/__init__.php

@@ -8,6 +8,8 @@
 
 phutil_require_module('phabricator', 'applications/diffusion/data/pathchange');
 phutil_require_module('phabricator', 'applications/diffusion/query/history/base');
+phutil_require_module('phabricator', 'applications/repository/storage/commit');
+phutil_require_module('phabricator', 'applications/repository/storage/commitdata');
 phutil_require_module('phabricator', 'applications/repository/storage/repository');
 phutil_require_module('phabricator', 'storage/queryfx');
 

src/applications/diffusion/view/historytable/__init__.php

@@ -8,7 +8,6 @@
 
 phutil_require_module('phabricator', 'applications/diffusion/view/base');
 phutil_require_module('phabricator', 'view/control/table');
-phutil_require_module('phabricator', 'view/utils');
 
 phutil_require_module('phutil', 'markup');
 

src/applications/repository/worker/commitchangeparser/svn/PhabricatorRepositorySvnCommitChangeParserWorker.php

@@ -516,7 +516,7 @@ class PhabricatorRepositorySvnCommitChangeParserWorker
       $parent = $repository_uri.$parent.'@'.$lookup['rawCommit'];
       $parent = escapeshellarg($parent);
       $parents[$parent] = true;
-      $path_mapping[$parent][] = $path;
+      $path_mapping[$parent][] = dirname($path);
     }
 
     $result_map = array();
@@ -592,21 +592,104 @@ class PhabricatorRepositorySvnCommitChangeParserWorker
     $rev  = $info['rawCommit'];
     $path = $this->encodeSVNPath($path);
 
-    // TODO: This is a scalability nightmare.
+    $hashkey = md5($repository->getDetail('remote-uri').$path.'@'.$rev);
 
-    list($raw_xml) = execx(
+    // This method is quite horrible. The underlying challenge is that some
-      'svn --non-interactive --xml ls -R %s%s@%d',
+    // commits in the Facebook repository are enormous, taking multiple hours
-      $repository->getDetail('remote-uri'),
+    // to 'ls -R' out of the repository and producing XML files >1GB in size.
-      $path,
-      $rev);
 
+    // If we try to SimpleXML them, the object exhausts available memory on a
+    // 64G machine. Instead, cache the XML output and then parse it line by line
+    // to limit space requirements.
+
+    $cache_loc = sys_get_temp_dir().'/diffusion.'.$hashkey.'.svnls';
+    if (!Filesystem::pathExists($cache_loc)) {
+      $tmp = new TempFile();
+      execx(
+        'svn --non-interactive --xml ls -R %s%s@%d > %s',
+        $repository->getDetail('remote-uri'),
+        $path,
+        $rev,
+        $tmp);
+      execx(
+        'mv %s %s',
+        $tmp,
+        $cache_loc);
+    }
+
+    $map = $this->parseRecursiveListFileData($cache_loc);
+    Filesystem::remove($cache_loc);
+
+    return $map;
+  }
+
+  private function parseRecursiveListFileData($file_path) {
     $map = array();
 
-    $xml = new SimpleXMLElement($raw_xml);
+    $mode = 'xml';
-    foreach ($xml->list[0] as $entry) {
+    $done = false;
-      $key = (string)$entry->name;
+    $entry = null;
-      $file_type = $this->getFileTypeFromSVNKind($entry['kind']);
+    foreach (new LinesOfALargeFile($file_path) as $lno => $line) {
-      $map[$key] = $file_type;
+      switch ($mode) {
+        case 'entry':
+          if ($line == '</entry>') {
+            $entry = implode('', $entry);
+            $pattern = '@^\s+kind="(file|dir)">'.
+                       '<name>(.*?)</name>'.
+                       '(<size>(.*?)</size>)?@';
+            $matches = null;
+            if (!preg_match($pattern, $entry, $matches)) {
+              throw new Exception("Unable to parse entry!");
+            }
+            $map[html_entity_decode($matches[2])] =
+              $this->getFileTypeFromSVNKind($matches[1]);
+            $mode = 'entry-or-end';
+          } else {
+            $entry[] = $line;
+          }
+          break;
+        case 'entry-or-end':
+          if ($line == '</list>') {
+            $done = true;
+            break 2;
+          } else if ($line == '<entry') {
+            $mode = 'entry';
+            $entry = array();
+          } else {
+            throw new Exception("Expected </list> or <entry, got {$line}.");
+          }
+          break;
+        case 'xml':
+          $expect = '<?xml version="1.0"?>';
+          if ($line !== $expect) {
+            throw new Exception("Expected '{$expect}', got {$line}.");
+          }
+          $mode = 'list';
+          break;
+        case 'list':
+          $expect = '<lists>';
+          if ($line !== $expect) {
+            throw new Exception("Expected '{$expect}', got {$line}.");
+          }
+          $mode = 'list1';
+          break;
+        case 'list1':
+          $expect = '<list';
+          if ($line !== $expect) {
+            throw new Exception("Expected '{$expect}', got {$line}.");
+          }
+          $mode = 'list2';
+          break;
+        case 'list2':
+          if (!preg_match('/^\s+path="/', $line)) {
+            throw new Exception("Expected '   path=...', got {$line}.");
+          }
+          $mode = 'entry-or-end';
+          break;
+      }
+    }
+    if (!$done) {
+      throw new Exception("Unexpected end of file.");
     }
 
     return $map;
@@ -635,10 +718,3 @@ class PhabricatorRepositorySvnCommitChangeParserWorker
   }
 
 }
-
-
-
-
-
-
-

src/applications/repository/worker/commitchangeparser/svn/__init__.php

@@ -13,6 +13,9 @@ phutil_require_module('phabricator', 'applications/repository/worker/commitchang
 phutil_require_module('phabricator', 'storage/qsprintf');
 phutil_require_module('phabricator', 'storage/queryfx');
 
+phutil_require_module('phutil', 'filesystem');
+phutil_require_module('phutil', 'filesystem/linesofalargefile');
+phutil_require_module('phutil', 'filesystem/tempfile');
 phutil_require_module('phutil', 'future/exec');
 phutil_require_module('phutil', 'utils');