1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-10 00:42:41 +01:00

Refactor repository reparse scripts to be more useful

Summary:
Splitting up D960 a bit, see that for context.

We currently have two scripts, "parse_one_commit.php" and
"reparse_all_commit_messages.php", but they're sort of silly and you can't do
certain things with them. Replace them with one script which is more flexible
and can do specific reparse steps on individual commits or entire repos.

I left the old scripts as stubs since I think there are some FB wiki docs and
stuff that mention them. I'll delete them in a month or whenever I remember or
something.

Test Plan: Ran "reparse.php" with various arguments, including vs-one-commit,
vs-repository, with --trace, and against different types of repos.

Reviewers: Makinde, jungejason, nh, tuomaspelkonen, aran

Reviewed By: jungejason

CC: aran, jungejason

Differential Revision: 964
This commit is contained in:
epriestley 2011-09-26 15:04:04 -07:00
parent 7b8b469da3
commit be26c6a5c1
4 changed files with 247 additions and 142 deletions

View file

@ -17,69 +17,7 @@
* limitations under the License.
*/
$root = dirname(dirname(dirname(__FILE__)));
require_once $root.'/scripts/__init_script__.php';
require_once $root.'/scripts/__init_env__.php';
if (empty($argv[1])) {
echo "usage: parse_one_commit.php <commit_name> [--herald]\n";
die(1);
}
$commit = isset($argv[1]) ? $argv[1] : null;
if (!$commit) {
throw new Exception("Provide a commit to parse!");
}
$matches = null;
if (!preg_match('/r([A-Z]+)([a-z0-9]+)/', $commit, $matches)) {
throw new Exception("Can't parse commit identifier!");
}
$repo = id(new PhabricatorRepository())->loadOneWhere(
'callsign = %s',
$matches[1]);
if (!$repo) {
throw new Exception("Unknown repository!");
}
$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
'repositoryID = %d AND commitIdentifier = %s',
$repo->getID(),
$matches[2]);
if (!$commit) {
throw new Exception('Unknown commit.');
}
$workers = array();
$spec = array(
'commitID' => $commit->getID(),
'only' => true,
);
switch ($repo->getVersionControlSystem()) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
$workers[] = new PhabricatorRepositoryGitCommitMessageParserWorker(
$spec);
$workers[] = new PhabricatorRepositoryGitCommitChangeParserWorker(
$spec);
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
$workers[] = new PhabricatorRepositorySvnCommitMessageParserWorker(
$spec);
$workers[] = new PhabricatorRepositorySvnCommitChangeParserWorker(
$spec);
break;
default:
throw new Exception("Unknown repository type!");
}
if (isset($argv[2]) && $argv[2] == '--herald') {
$workers[] = new PhabricatorRepositoryCommitHeraldWorker($spec);
}
foreach ($workers as $worker) {
echo "Running ".get_class($worker)."...\n";
$worker->doWork();
}
echo "Done.\n";
echo "This script is obsolete. Instead, use:\n\n".
" $ reparse.php <commit_name> --message --change\n\n".
"See that script for more options.\n";
exit(1);

236
scripts/repository/reparse.php Executable file
View file

@ -0,0 +1,236 @@
#!/usr/bin/env php
<?php
/*
* Copyright 2011 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
$root = dirname(dirname(dirname(__FILE__)));
require_once $root.'/scripts/__init_script__.php';
require_once $root.'/scripts/__init_env__.php';
phutil_require_module('phutil', 'console');
$is_all = false;
$reparse_message = false;
$reparse_change = false;
$reparse_herald = false;
$reparse_what = false;
$args = array_slice($argv, 1);
foreach ($args as $arg) {
if (!strncmp($arg, '--', 2)) {
$flag = substr($arg, 2);
switch ($flag) {
case 'all':
$is_all = true;
break;
case 'message':
case 'messages':
$reparse_message = true;
break;
case 'change':
case 'changes':
$reparse_change = true;
break;
case 'herald':
$reparse_herald = true;
break;
case 'trace':
PhutilServiceProfiler::installEchoListener();
break;
case 'help':
help();
break;
default:
usage("Unknown flag '{$arg}'.");
}
} else {
if ($reparse_what) {
usage("Specify exactly one thing to reparse.");
}
$reparse_what = $arg;
}
}
if (!$reparse_what) {
usage("Specify a commit or repository to reparse.");
}
if (!$reparse_message && !$reparse_change && !$reparse_herald) {
usage("Specify what information to reparse with --message, --change, and/or ".
"--herald.");
}
$commits = array();
if ($is_all) {
$repository = id(new PhabricatorRepository())->loadOneWhere(
'callsign = %s OR phid = %s',
$reparse_what,
$reparse_what);
if (!$repository) {
throw new Exception("Unknown repository '{$reparse_what}'!");
}
$commits = id(new PhabricatorRepositoryCommit())->loadAllWhere(
'repositoryID = %d',
$repository->getID());
if (!$commits) {
throw new Exception("No commits have been discovered in that repository!");
}
$callsign = $repository->getCallsign();
} else {
$matches = null;
if (!preg_match('/r([A-Z]+)([a-z0-9]+)/', $reparse_what, $matches)) {
throw new Exception("Can't parse commit identifier!");
}
$callsign = $matches[1];
$commit_identifier = $matches[2];
$repository = id(new PhabricatorRepository())->loadOneWhere(
'callsign = %s',
$callsign);
if (!$repository) {
throw new Exception("No repository with callsign '{$callsign}'!");
}
$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
'repositoryID = %d AND commitIdentifier = %s',
$repository->getID(),
$commit_identifier);
if (!$commit) {
throw new Exception(
"No matching commit '{$commit_identifier}' in repository '{$callsign}'. ".
"(For git and mercurial repositories, you must specify the entire ".
"commit hash.)");
}
$commits = array($commit);
}
if ($is_all) {
echo phutil_console_format(
'**NOTE**: This script will queue tasks to reparse the data. Once the '.
'tasks have been queued, you need to run Taskmaster daemons to execute '.
'them.');
echo "\n\n";
echo "QUEUEING TASKS (".number_format(count($commits))." Commits):\n";
}
$tasks = array();
foreach ($commits as $commit) {
$classes = array();
switch ($repository->getVersionControlSystem()) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
if ($reparse_message) {
$classes[] = 'PhabricatorRepositoryGitCommitMessageParserWorker';
}
if ($reparse_change) {
$classes[] = 'PhabricatorRepositoryGitCommitChangeParserWorker';
}
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
if ($reparse_message) {
$classes[] = 'PhabricatorRepositoryMercurialCommitMessageParserWorker';
}
if ($reparse_change) {
$classes[] = 'PhabricatorRepositoryMercurialCommitChangeParserWorker';
}
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
if ($reparse_message) {
$classes[] = 'PhabricatorRepositorySvnCommitMessageParserWorker';
}
if ($reparse_change) {
$classes[] = 'PhabricatorRepositorySvnCommitChangeParserWorker';
}
break;
}
if ($reparse_herald) {
$classes[] = 'PhabricatorRepositoryCommitHeraldWorker';
}
$spec = array(
'commitID' => $commit->getID(),
'only' => true,
);
if ($is_all) {
foreach ($classes as $class) {
$task = new PhabricatorWorkerTask();
$task->setTaskClass($class);
$task->setData($spec);
$task->save();
$commit_name = 'r'.$callsign.$commit->getCommitIdentifier();
echo " Queued '{$class}' for commit '{$commit_name}'.\n";
}
} else {
foreach ($classes as $class) {
$worker = newv($class, array($spec));
echo "Running '{$class}'...\n";
$worker->doWork();
}
}
}
echo "\nDone.\n";
function usage($message) {
echo "Usage Error: {$message}";
echo "\n\n";
echo "Run 'reparse.php --help' for detailed help.\n";
exit(1);
}
function help() {
$help = <<<EOHELP
**SUMMARY**
**reparse.php** __what__ __which_parts__ [--trace]
Rerun the Diffusion parser on specific commits and repositories. Mostly
useful for debugging changes to Diffusion.
__what__: what to reparse
__commit__
Reparse one commit. This mode will reparse the commit in-process.
--all __repository_callsign__
--all __repository_phid__
Reparse all commits in the specified repository. These modes queue
parsers into the task queue, you must run taskmasters to actually
do the parses them.
__which_parts__: which parts of the thing to reparse
__--message__
Reparse commit messages.
__--change__
Reparse changes.
__--herald__
Reevaluate Herald rules (may send huge amounts of email!)
__--trace__: run with debug tracing
__--help__: show this help
**EXAMPLES**
reparse.php rX123 --change # Reparse change for "rX123".
reparse.php --all E --message # Reparse all messages in "E" repository.
EOHELP;
echo phutil_console_format($help);
exit(1);
}

View file

@ -17,79 +17,8 @@
* limitations under the License.
*/
$root = dirname(dirname(dirname(__FILE__)));
require_once $root.'/scripts/__init_script__.php';
require_once $root.'/scripts/__init_env__.php';
echo "This script is obsolete. Instead, use:\n\n".
" $ reparse.php --all <repository_name> --message\n\n".
"See that script for more options.\n";
exit(1);
phutil_require_module('phutil', 'console');
if (empty($argv[1])) {
echo "usage: reparse_all_commit_messages.php all\n".
" reparse_all_commit_messages.php <repository_callsign>\n";
exit(1);
}
echo phutil_console_format(
'This script will queue tasks to reparse every commit message known to '.
'Diffusion. Once the tasks have been inserted, you need to start '.
'Taskmaster daemons to execute them.');
$ok = phutil_console_confirm('Do you want to continue?');
if (!$ok) {
die(1);
}
if ($argv[1] == 'all') {
echo "Loading all repositories...\n";
$repositories = id(new PhabricatorRepository())->loadAll();
echo "Loading all commits...\n";
$commits = id(new PhabricatorRepositoryCommit())->loadAll();
} else {
$callsign = $argv[1];
echo "Loading '{$callsign}' repository...\n";
$repository = id(new PhabricatorRepository())->loadOneWhere(
'callsign = %s',
$argv[1]);
if (!$repository) {
throw new Exception("No such repository exists!");
}
$repositories = array(
$repository->getID() => $repository,
);
echo "Loading commits in '{$callsign}' repository...\n";
$commits = id(new PhabricatorRepositoryCommit())->loadAllWhere(
'repositoryID = %d',
$repository->getID());
}
echo "Inserting tasks for ".count($commits)." commits";
foreach ($commits as $commit) {
echo ".";
$id = $commit->getID();
$repo = idx($repositories, $commit->getRepositoryID());
if (!$repo) {
echo "\nWarning: Commit #{$id} has an invalid repository ID.\n";
}
switch ($repo->getVersionControlSystem()) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
$task_class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
$task_class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
break;
default:
throw new Exception("Unknown repository type!");
}
$task = new PhabricatorWorkerTask();
$task->setTaskClass($task_class);
$task->setData(
array(
'commitID' => $commit->getID(),
'only' => true,
));
$task->save();
}
echo "\nDone.\n";

View file

@ -62,10 +62,12 @@ class PhabricatorWorkerTaskDetailController
"<strong>NOTE:</strong> ".
"You can manually retry this task by running this script:".
"<pre>".
"phabricator/\$ ./scripts/repository/parse_one_commit.php ".
"phabricator/\$ ./scripts/repository/reparse.php ".
"r".
phutil_escape_html($repository->getCallsign()).
phutil_escape_html($commit->getCommitIdentifier()).
" ".
"--change".
"</pre>";
}
}