Refactor repository reparse scripts to be more useful

Summary:
Splitting up D960 a bit, see that for context.

We currently have two scripts, "parse_one_commit.php" and
"reparse_all_commit_messages.php", but they're sort of silly and you can't do
certain things with them. Replace them with one script which is more flexible
and can do specific reparse steps on individual commits or entire repos.

I left the old scripts as stubs since I think there are some FB wiki docs and
stuff that mention them. I'll delete them in a month or whenever I remember or
something.

Test Plan: Ran "reparse.php" with various arguments, including vs-one-commit,
vs-repository, with --trace, and against different types of repos.

Reviewers: Makinde, jungejason, nh, tuomaspelkonen, aran

Reviewed By: jungejason

CC: aran, jungejason

Differential Revision: 964

scripts/repository/parse_one_commit.php

@@ -17,69 +17,7 @@
  * limitations under the License.
  */
 
-$root = dirname(dirname(dirname(__FILE__)));
+echo "This script is obsolete. Instead, use:\n\n".
-require_once $root.'/scripts/__init_script__.php';
+     "  $ reparse.php <commit_name> --message --change\n\n".
-require_once $root.'/scripts/__init_env__.php';
+     "See that script for more options.\n";
-
+exit(1);
-if (empty($argv[1])) {
-  echo "usage: parse_one_commit.php <commit_name> [--herald]\n";
-  die(1);
-}
-
-$commit = isset($argv[1]) ? $argv[1] : null;
-if (!$commit) {
-  throw new Exception("Provide a commit to parse!");
-}
-$matches = null;
-if (!preg_match('/r([A-Z]+)([a-z0-9]+)/', $commit, $matches)) {
-  throw new Exception("Can't parse commit identifier!");
-}
-$repo = id(new PhabricatorRepository())->loadOneWhere(
-  'callsign = %s',
-  $matches[1]);
-if (!$repo) {
-  throw new Exception("Unknown repository!");
-}
-$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
-  'repositoryID = %d AND commitIdentifier = %s',
-  $repo->getID(),
-  $matches[2]);
-if (!$commit) {
-  throw new Exception('Unknown commit.');
-}
-
-$workers = array();
-
-$spec = array(
-  'commitID'  => $commit->getID(),
-  'only'      => true,
-);
-
-switch ($repo->getVersionControlSystem()) {
-  case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
-    $workers[] = new PhabricatorRepositoryGitCommitMessageParserWorker(
-      $spec);
-    $workers[] = new PhabricatorRepositoryGitCommitChangeParserWorker(
-      $spec);
-    break;
-  case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
-    $workers[] = new PhabricatorRepositorySvnCommitMessageParserWorker(
-      $spec);
-    $workers[] = new PhabricatorRepositorySvnCommitChangeParserWorker(
-      $spec);
-    break;
-  default:
-    throw new Exception("Unknown repository type!");
-}
-
-if (isset($argv[2]) && $argv[2] == '--herald') {
-  $workers[] = new PhabricatorRepositoryCommitHeraldWorker($spec);
-}
-
-foreach ($workers as $worker) {
-  echo "Running ".get_class($worker)."...\n";
-  $worker->doWork();
-}
-
-echo "Done.\n";
-

scripts/repository/reparse.php

@@ -0,0 +1,236 @@
+#!/usr/bin/env php
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$root = dirname(dirname(dirname(__FILE__)));
+require_once $root.'/scripts/__init_script__.php';
+require_once $root.'/scripts/__init_env__.php';
+
+phutil_require_module('phutil', 'console');
+
+$is_all = false;
+$reparse_message = false;
+$reparse_change = false;
+$reparse_herald = false;
+$reparse_what = false;
+
+$args = array_slice($argv, 1);
+foreach ($args as $arg) {
+  if (!strncmp($arg, '--', 2)) {
+    $flag = substr($arg, 2);
+    switch ($flag) {
+      case 'all':
+        $is_all = true;
+        break;
+      case 'message':
+      case 'messages':
+        $reparse_message = true;
+        break;
+      case 'change':
+      case 'changes':
+        $reparse_change = true;
+        break;
+      case 'herald':
+        $reparse_herald = true;
+        break;
+      case 'trace':
+        PhutilServiceProfiler::installEchoListener();
+        break;
+      case 'help':
+        help();
+        break;
+      default:
+        usage("Unknown flag '{$arg}'.");
+    }
+  } else {
+    if ($reparse_what) {
+      usage("Specify exactly one thing to reparse.");
+    }
+    $reparse_what = $arg;
+  }
+}
+
+if (!$reparse_what) {
+  usage("Specify a commit or repository to reparse.");
+}
+if (!$reparse_message && !$reparse_change && !$reparse_herald) {
+  usage("Specify what information to reparse with --message, --change, and/or ".
+        "--herald.");
+}
+
+$commits = array();
+if ($is_all) {
+  $repository = id(new PhabricatorRepository())->loadOneWhere(
+    'callsign = %s OR phid = %s',
+    $reparse_what,
+    $reparse_what);
+  if (!$repository) {
+    throw new Exception("Unknown repository '{$reparse_what}'!");
+  }
+  $commits = id(new PhabricatorRepositoryCommit())->loadAllWhere(
+    'repositoryID = %d',
+    $repository->getID());
+  if (!$commits) {
+    throw new Exception("No commits have been discovered in that repository!");
+  }
+  $callsign = $repository->getCallsign();
+} else {
+  $matches = null;
+  if (!preg_match('/r([A-Z]+)([a-z0-9]+)/', $reparse_what, $matches)) {
+    throw new Exception("Can't parse commit identifier!");
+  }
+  $callsign = $matches[1];
+  $commit_identifier = $matches[2];
+  $repository = id(new PhabricatorRepository())->loadOneWhere(
+    'callsign = %s',
+    $callsign);
+  if (!$repository) {
+    throw new Exception("No repository with callsign '{$callsign}'!");
+  }
+  $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
+    'repositoryID = %d AND commitIdentifier = %s',
+    $repository->getID(),
+    $commit_identifier);
+  if (!$commit) {
+    throw new Exception(
+      "No matching commit '{$commit_identifier}' in repository '{$callsign}'. ".
+      "(For git and mercurial repositories, you must specify the entire ".
+      "commit hash.)");
+  }
+  $commits = array($commit);
+}
+
+if ($is_all) {
+  echo phutil_console_format(
+    '**NOTE**: This script will queue tasks to reparse the data. Once the '.
+    'tasks have been queued, you need to run Taskmaster daemons to execute '.
+    'them.');
+  echo "\n\n";
+  echo "QUEUEING TASKS (".number_format(count($commits))." Commits):\n";
+}
+
+$tasks = array();
+foreach ($commits as $commit) {
+  $classes = array();
+  switch ($repository->getVersionControlSystem()) {
+    case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
+      if ($reparse_message) {
+        $classes[] = 'PhabricatorRepositoryGitCommitMessageParserWorker';
+      }
+      if ($reparse_change) {
+        $classes[] = 'PhabricatorRepositoryGitCommitChangeParserWorker';
+      }
+      break;
+    case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
+      if ($reparse_message) {
+        $classes[] = 'PhabricatorRepositoryMercurialCommitMessageParserWorker';
+      }
+      if ($reparse_change) {
+        $classes[] = 'PhabricatorRepositoryMercurialCommitChangeParserWorker';
+      }
+      break;
+    case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
+      if ($reparse_message) {
+        $classes[] = 'PhabricatorRepositorySvnCommitMessageParserWorker';
+      }
+      if ($reparse_change) {
+        $classes[] = 'PhabricatorRepositorySvnCommitChangeParserWorker';
+      }
+      break;
+  }
+
+  if ($reparse_herald) {
+    $classes[] = 'PhabricatorRepositoryCommitHeraldWorker';
+  }
+
+  $spec = array(
+    'commitID'  => $commit->getID(),
+    'only'      => true,
+  );
+
+  if ($is_all) {
+    foreach ($classes as $class) {
+      $task = new PhabricatorWorkerTask();
+      $task->setTaskClass($class);
+      $task->setData($spec);
+      $task->save();
+
+      $commit_name = 'r'.$callsign.$commit->getCommitIdentifier();
+      echo "  Queued '{$class}' for commit '{$commit_name}'.\n";
+    }
+  } else {
+    foreach ($classes as $class) {
+      $worker = newv($class, array($spec));
+      echo "Running '{$class}'...\n";
+      $worker->doWork();
+    }
+  }
+}
+
+echo "\nDone.\n";
+
+function usage($message) {
+  echo "Usage Error: {$message}";
+  echo "\n\n";
+  echo "Run 'reparse.php --help' for detailed help.\n";
+  exit(1);
+}
+
+function help() {
+  $help = <<<EOHELP
+**SUMMARY**
+
+    **reparse.php** __what__ __which_parts__ [--trace]
+
+    Rerun the Diffusion parser on specific commits and repositories. Mostly
+    useful for debugging changes to Diffusion.
+
+    __what__: what to reparse
+
+        __commit__
+            Reparse one commit. This mode will reparse the commit in-process.
+
+        --all __repository_callsign__
+        --all __repository_phid__
+            Reparse all commits in the specified repository. These modes queue
+            parsers into the task queue, you must run taskmasters to actually
+            do the parses them.
+
+    __which_parts__: which parts of the thing to reparse
+
+        __--message__
+            Reparse commit messages.
+
+        __--change__
+            Reparse changes.
+
+        __--herald__
+            Reevaluate Herald rules (may send huge amounts of email!)
+
+    __--trace__: run with debug tracing
+    __--help__: show this help
+
+**EXAMPLES**
+
+  reparse.php rX123 --change       # Reparse change for "rX123".
+  reparse.php --all E --message    # Reparse all messages in "E" repository.
+
+EOHELP;
+  echo phutil_console_format($help);
+  exit(1);
+}

scripts/repository/reparse_all_commit_messages.php

@@ -17,79 +17,8 @@
  * limitations under the License.
  */
 
-$root = dirname(dirname(dirname(__FILE__)));
+echo "This script is obsolete. Instead, use:\n\n".
-require_once $root.'/scripts/__init_script__.php';
+     "  $ reparse.php --all <repository_name> --message\n\n".
-require_once $root.'/scripts/__init_env__.php';
+     "See that script for more options.\n";
+exit(1);
 
-phutil_require_module('phutil', 'console');
-
-if (empty($argv[1])) {
-  echo "usage: reparse_all_commit_messages.php all\n".
-       "       reparse_all_commit_messages.php <repository_callsign>\n";
-  exit(1);
-}
-
-
-echo phutil_console_format(
-  'This script will queue tasks to reparse every commit message known to '.
-  'Diffusion. Once the tasks have been inserted, you need to start '.
-  'Taskmaster daemons to execute them.');
-
-$ok = phutil_console_confirm('Do you want to continue?');
-if (!$ok) {
-  die(1);
-}
-
-if ($argv[1] == 'all') {
-  echo "Loading all repositories...\n";
-  $repositories = id(new PhabricatorRepository())->loadAll();
-  echo "Loading all commits...\n";
-  $commits = id(new PhabricatorRepositoryCommit())->loadAll();
-} else {
-  $callsign = $argv[1];
-  echo "Loading '{$callsign}' repository...\n";
-  $repository = id(new PhabricatorRepository())->loadOneWhere(
-    'callsign = %s',
-    $argv[1]);
-  if (!$repository) {
-    throw new Exception("No such repository exists!");
-  }
-  $repositories = array(
-    $repository->getID() => $repository,
-  );
-  echo "Loading commits in '{$callsign}' repository...\n";
-  $commits = id(new PhabricatorRepositoryCommit())->loadAllWhere(
-    'repositoryID = %d',
-    $repository->getID());
-}
-
-echo "Inserting tasks for ".count($commits)." commits";
-foreach ($commits as $commit) {
-  echo ".";
-  $id = $commit->getID();
-  $repo = idx($repositories, $commit->getRepositoryID());
-  if (!$repo) {
-    echo "\nWarning: Commit #{$id} has an invalid repository ID.\n";
-  }
-
-  switch ($repo->getVersionControlSystem()) {
-    case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
-      $task_class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
-      break;
-    case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
-      $task_class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
-      break;
-    default:
-      throw new Exception("Unknown repository type!");
-  }
-
-  $task = new PhabricatorWorkerTask();
-  $task->setTaskClass($task_class);
-  $task->setData(
-    array(
-      'commitID'  => $commit->getID(),
-      'only'      => true,
-    ));
-  $task->save();
-}
-echo "\nDone.\n";

src/applications/daemon/controller/workertaskdetail/PhabricatorWorkerTaskDetailController.php

@@ -62,10 +62,12 @@ class PhabricatorWorkerTaskDetailController
                 "<strong>NOTE:</strong> ".
                 "You can manually retry this task by running this script:".
                 "<pre>".
-                  "phabricator/\$ ./scripts/repository/parse_one_commit.php ".
+                  "phabricator/\$ ./scripts/repository/reparse.php ".
                   "r".
                   phutil_escape_html($repository->getCallsign()).
                   phutil_escape_html($commit->getCommitIdentifier()).
+                  " ".
+                  "--change".
                 "</pre>";
             }
           }