mirror of
https://we.phorge.it/source/phorge.git
synced 2024-11-21 22:32:41 +01:00
Move symbols to be repository-based
Summary: Fixes T7220. Ref T7977. Changes symbols from being bound to an Arcanist project to being bound to a repository. Test Plan: - Added symbols and then applied migrations, symbols seemed to be migrated successfully. - Tested the `/diffusion/symbol/$SYMBOL_NAME` endpoint. - Tested the `/diffusion/symbol/$SYMBOL_NAME` endpoint with the `?repositories=$REPOSITORY_PHID` parameter. Reviewers: #blessed_reviewers, epriestley Reviewed By: #blessed_reviewers, epriestley Subscribers: avivey, Korvin, epriestley Maniphest Tasks: T7977, T7220 Differential Revision: https://secure.phabricator.com/D12608
This commit is contained in:
parent
38e89fbb08
commit
2483f6f120
16 changed files with 419 additions and 401 deletions
|
@ -0,0 +1,2 @@
|
|||
ALTER TABLE {$NAMESPACE}_repository.repository_symbol
|
||||
ADD repositoryPHID varbinary(64) NOT NULL AFTER arcanistProjectID;
|
26
resources/sql/autopatches/20150503.repositorysymbols.2.php
Normal file
26
resources/sql/autopatches/20150503.repositorysymbols.2.php
Normal file
|
@ -0,0 +1,26 @@
|
|||
<?php
|
||||
|
||||
$projects = id(new PhabricatorRepositoryArcanistProjectQuery())
|
||||
->setViewer(PhabricatorUser::getOmnipotentUser())
|
||||
->needRepositories(true)
|
||||
->execute();
|
||||
|
||||
$table = new PhabricatorRepositorySymbol();
|
||||
$conn_w = $table->establishConnection('w');
|
||||
|
||||
foreach ($projects as $project) {
|
||||
$repo = $project->getRepository();
|
||||
|
||||
if (!$repo) {
|
||||
continue;
|
||||
}
|
||||
|
||||
echo pht("Migrating symbols for '%s' project...\n", $project->getName());
|
||||
|
||||
queryfx(
|
||||
$conn_w,
|
||||
'UPDATE %T SET repositoryPHID = %s WHERE arcanistProjectID = %d',
|
||||
$table->getTableName(),
|
||||
$repo->getPHID(),
|
||||
$project->getID());
|
||||
}
|
|
@ -0,0 +1,2 @@
|
|||
ALTER TABLE {$NAMESPACE}_repository.repository_symbol
|
||||
DROP COLUMN arcanistProjectID;
|
|
@ -1,32 +0,0 @@
|
|||
#!/usr/bin/env php
|
||||
<?php
|
||||
|
||||
$root = dirname(dirname(dirname(__FILE__)));
|
||||
require_once $root.'/scripts/__init_script__.php';
|
||||
|
||||
$project = id(new PhabricatorRepositoryArcanistProject())->loadOneWhere(
|
||||
'name = %s', $argv[1]);
|
||||
if (!$project) {
|
||||
throw new Exception('No such arcanist project.');
|
||||
}
|
||||
|
||||
$input = file_get_contents('php://stdin');
|
||||
$normalized = array();
|
||||
foreach (explode("\n", trim($input)) as $path) {
|
||||
// emulate the behavior of the symbol generation scripts
|
||||
$normalized[] = '/'.ltrim($path, './');
|
||||
}
|
||||
$paths = PhabricatorRepositoryCommitChangeParserWorker::lookupOrCreatePaths(
|
||||
$normalized);
|
||||
|
||||
$symbol = new PhabricatorRepositorySymbol();
|
||||
$conn_w = $symbol->establishConnection('w');
|
||||
|
||||
foreach (array_chunk(array_values($paths), 128) as $chunk) {
|
||||
queryfx(
|
||||
$conn_w,
|
||||
'DELETE FROM %T WHERE arcanistProjectID = %d AND pathID IN (%Ld)',
|
||||
$symbol->getTableName(),
|
||||
$project->getID(),
|
||||
$chunk);
|
||||
}
|
58
scripts/symbols/clear_repository_symbols.php
Executable file
58
scripts/symbols/clear_repository_symbols.php
Executable file
|
@ -0,0 +1,58 @@
|
|||
#!/usr/bin/env php
|
||||
<?php
|
||||
|
||||
$root = dirname(dirname(dirname(__FILE__)));
|
||||
require_once $root.'/scripts/__init_script__.php';
|
||||
|
||||
$args = new PhutilArgumentParser($argv);
|
||||
$args->setSynopsis(<<<EOSYNOPSIS
|
||||
**clear_repository_symbols.php** [__options__] __callsign__
|
||||
|
||||
Clear repository symbols.
|
||||
EOSYNOPSIS
|
||||
);
|
||||
$args->parseStandardArguments();
|
||||
$args->parse(
|
||||
array(
|
||||
array(
|
||||
'name' => 'callsign',
|
||||
'wildcard' => true,
|
||||
),
|
||||
));
|
||||
|
||||
$callsigns = $args->getArg('callsign');
|
||||
if (count($callsigns) !== 1) {
|
||||
$args->printHelpAndExit();
|
||||
}
|
||||
|
||||
$callsign = head($callsigns);
|
||||
$repository = id(new PhabricatorRepositoryQuery())
|
||||
->setViewer(PhabricatorUser::getOmnipotentUser())
|
||||
->withCallsigns($callsigns)
|
||||
->executeOne();
|
||||
|
||||
if (!$repository) {
|
||||
echo pht("Repository '%s' does not exist.", $callsign);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
$input = file_get_contents('php://stdin');
|
||||
$normalized = array();
|
||||
foreach (explode("\n", trim($input)) as $path) {
|
||||
// Emulate the behavior of the symbol generation scripts.
|
||||
$normalized[] = '/'.ltrim($path, './');
|
||||
}
|
||||
$paths = PhabricatorRepositoryCommitChangeParserWorker::lookupOrCreatePaths(
|
||||
$normalized);
|
||||
|
||||
$symbol = new PhabricatorRepositorySymbol();
|
||||
$conn_w = $symbol->establishConnection('w');
|
||||
|
||||
foreach (array_chunk(array_values($paths), 128) as $chunk) {
|
||||
queryfx(
|
||||
$conn_w,
|
||||
'DELETE FROM %T WHERE repositoryPHID = %s AND pathID IN (%Ld)',
|
||||
$symbol->getTableName(),
|
||||
$repository->getPHID(),
|
||||
$chunk);
|
||||
}
|
|
@ -4,79 +4,94 @@
|
|||
$root = dirname(dirname(dirname(__FILE__)));
|
||||
require_once $root.'/scripts/__init_script__.php';
|
||||
|
||||
$args = new PhutilArgumentParser($argv);
|
||||
$args->setSynopsis(<<<EOSYNOPSIS
|
||||
**generate_ctags_symbols.php** [__options__]
|
||||
|
||||
Generate repository symbols using Exuberant Ctags. Paths are read from stdin.
|
||||
EOSYNOPSIS
|
||||
);
|
||||
$args->parseStandardArguments();
|
||||
|
||||
if (ctags_check_executable() == false) {
|
||||
echo phutil_console_format(
|
||||
"Could not find Exuberant ctags. Make sure it is installed and\n".
|
||||
"available in executable path.\n\n".
|
||||
"Exuberant ctags project page: http://ctags.sourceforge.net/\n");
|
||||
"%s\n\n%s\n",
|
||||
pht(
|
||||
'Could not find Exuberant Ctags. Make sure it is installed and '.
|
||||
'available in executable path.'),
|
||||
pht(
|
||||
'Exuberant Ctags project page: %s',
|
||||
'http://ctags.sourceforge.net/'));
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if ($argc !== 1 || posix_isatty(STDIN)) {
|
||||
if (posix_isatty(STDIN)) {
|
||||
echo phutil_console_format(
|
||||
"usage: find . -type f -name '*.py' | ./generate_ctags_symbols.php\n");
|
||||
"%s\n",
|
||||
pht(
|
||||
'Usage: %s',
|
||||
"find . -type f -name '*.py' | ./generate_ctags_symbols.php"));
|
||||
exit(1);
|
||||
}
|
||||
|
||||
$input = file_get_contents('php://stdin');
|
||||
$input = trim($input);
|
||||
$input = explode("\n", $input);
|
||||
|
||||
$data = array();
|
||||
$futures = array();
|
||||
|
||||
foreach ($input as $file) {
|
||||
foreach (explode("\n", trim($input)) as $file) {
|
||||
$file = Filesystem::readablePath($file);
|
||||
$futures[$file] = ctags_get_parser_future($file);
|
||||
}
|
||||
|
||||
$futures = id(new FutureIterator($futures))
|
||||
->limit(8);
|
||||
foreach ($futures as $file => $future) {
|
||||
$futures = new FutureIterator($futures);
|
||||
foreach ($futures->limit(8) as $file => $future) {
|
||||
$tags = $future->resolve();
|
||||
$tags = explode("\n", $tags[1]);
|
||||
|
||||
foreach ($tags as $tag) {
|
||||
$parts = explode(';', $tag);
|
||||
// skip lines that we can not parse
|
||||
|
||||
// Skip lines that we can not parse.
|
||||
if (count($parts) < 2) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// split ctags information
|
||||
// Split ctags information.
|
||||
$tag_info = explode("\t", $parts[0]);
|
||||
// split exuberant ctags "extension fields" (additional information)
|
||||
|
||||
// Split exuberant ctags "extension fields" (additional information).
|
||||
$parts[1] = trim($parts[1], "\t \"");
|
||||
$extension_fields = explode("\t", $parts[1]);
|
||||
|
||||
// skip lines that we can not parse
|
||||
// Skip lines that we can not parse.
|
||||
if (count($tag_info) < 3 || count($extension_fields) < 2) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// default $context to empty
|
||||
// Default context to empty.
|
||||
$extension_fields[] = '';
|
||||
list($token, $file_path, $line_num) = $tag_info;
|
||||
list($type, $language, $context) = $extension_fields;
|
||||
|
||||
// skip lines with tokens containing a space
|
||||
// Skip lines with tokens containing a space.
|
||||
if (strpos($token, ' ') !== false) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// strip "language:"
|
||||
// Strip "language:"
|
||||
$language = substr($language, 9);
|
||||
|
||||
// To keep consistent with "Separate with commas, for example: php, py"
|
||||
// in Arcanist Project edit form.
|
||||
$language = str_ireplace('python', 'py', $language);
|
||||
|
||||
// also, "normalize" c++ and c#
|
||||
// Also, "normalize" C++ and C#.
|
||||
$language = str_ireplace('c++', 'cpp', $language);
|
||||
$language = str_ireplace('c#', 'cs', $language);
|
||||
|
||||
// Ruby has "singleton method", for example
|
||||
// Ruby has "singleton method", for example.
|
||||
$type = substr(str_replace(' ', '_', $type), 0, 12);
|
||||
|
||||
// class:foo, struct:foo, union:foo, enum:foo, ...
|
||||
$context = last(explode(':', $context, 2));
|
||||
|
||||
|
@ -89,25 +104,18 @@ foreach ($futures as $file => $future) {
|
|||
}
|
||||
}
|
||||
|
||||
function ctags_get_parser_future($file_path) {
|
||||
$future = new ExecFuture('ctags -n --fields=Kls -o - %s',
|
||||
$file_path);
|
||||
function ctags_get_parser_future($path) {
|
||||
$future = new ExecFuture('ctags -n --fields=Kls -o - %s', $path);
|
||||
return $future;
|
||||
}
|
||||
|
||||
function ctags_check_executable() {
|
||||
$future = new ExecFuture('ctags --version');
|
||||
$result = $future->resolve();
|
||||
|
||||
if (empty($result[1])) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
$result = exec_manual('ctags --version');
|
||||
return !empty($result[1]);
|
||||
}
|
||||
|
||||
function print_symbol($file, $line_num, $type, $token, $context, $language) {
|
||||
// get rid of relative path
|
||||
// Get rid of relative path.
|
||||
$file = explode('/', $file);
|
||||
if ($file[0] == '.' || $file[0] == '..') {
|
||||
array_shift($file);
|
||||
|
|
|
@ -4,28 +4,36 @@
|
|||
$root = dirname(dirname(dirname(__FILE__)));
|
||||
require_once $root.'/scripts/__init_script__.php';
|
||||
|
||||
if ($argc !== 1 || posix_isatty(STDIN)) {
|
||||
$args = new PhutilArgumentParser($argv);
|
||||
$args->setSynopsis(<<<EOSYNOPSIS
|
||||
**generate_php_symbols.php** [__options__]
|
||||
|
||||
Generate repository symbols using XHPAST. Paths are read from stdin.
|
||||
EOSYNOPSIS
|
||||
);
|
||||
$args->parseStandardArguments();
|
||||
|
||||
if (posix_isatty(STDIN)) {
|
||||
echo phutil_console_format(
|
||||
"usage: find . -type f -name '*.php' | ./generate_php_symbols.php\n");
|
||||
"%s\n",
|
||||
pht(
|
||||
'Usage: %s',
|
||||
"find . -type f -name '*.php' | ./generate_php_symbols.php"));
|
||||
exit(1);
|
||||
}
|
||||
|
||||
$input = file_get_contents('php://stdin');
|
||||
$input = trim($input);
|
||||
$input = explode("\n", $input);
|
||||
|
||||
$data = array();
|
||||
$futures = array();
|
||||
|
||||
foreach ($input as $file) {
|
||||
foreach (explode("\n", trim($input)) as $file) {
|
||||
$file = Filesystem::readablePath($file);
|
||||
$data[$file] = Filesystem::readFile($file);
|
||||
$futures[$file] = PhutilXHPASTBinary::getParserFuture($data[$file]);
|
||||
}
|
||||
|
||||
$futures = id(new FutureIterator($futures))
|
||||
->limit(8);
|
||||
foreach ($futures as $file => $future) {
|
||||
$futures = new FutureIterator($futures);
|
||||
foreach ($futures->limit(8) as $file => $future) {
|
||||
$tree = XHPASTTree::newFromDataAndResolvedExecFuture(
|
||||
$data[$file],
|
||||
$future->resolve());
|
||||
|
@ -36,7 +44,7 @@ foreach ($futures as $file => $future) {
|
|||
$functions = $root->selectDescendantsOfType('n_FUNCTION_DECLARATION');
|
||||
foreach ($functions as $function) {
|
||||
$name = $function->getChildByIndex(2);
|
||||
// Skip anonymous functions
|
||||
// Skip anonymous functions.
|
||||
if (!$name->getConcreteString()) {
|
||||
continue;
|
||||
}
|
||||
|
@ -67,8 +75,8 @@ foreach ($futures as $file => $future) {
|
|||
}
|
||||
|
||||
foreach ($scopes as $scope) {
|
||||
// this prints duplicate symbols in the case of nested classes
|
||||
// luckily, PHP doesn't allow those
|
||||
// This prints duplicate symbols in the case of nested classes.
|
||||
// Luckily, PHP doesn't allow those.
|
||||
list($class, $class_name) = $scope;
|
||||
|
||||
$consts = $class->selectDescendantsOfType(
|
||||
|
@ -100,15 +108,15 @@ foreach ($futures as $file => $future) {
|
|||
}
|
||||
}
|
||||
|
||||
function print_symbol($file, $type, $token, $context = null) {
|
||||
function print_symbol($file, $type, XHPASTNode $node, $context = null) {
|
||||
$parts = array(
|
||||
$context ? $context->getConcreteString() : '',
|
||||
// variable tokens are `$name`, not just `name`, so strip the $ off of
|
||||
// Variable tokens are `$name`, not just `name`, so strip the "$"" off of
|
||||
// class field names
|
||||
ltrim($token->getConcreteString(), '$'),
|
||||
ltrim($node->getConcreteString(), '$'),
|
||||
$type,
|
||||
'php',
|
||||
$token->getLineNumber(),
|
||||
$node->getLineNumber(),
|
||||
'/'.ltrim($file, './'),
|
||||
);
|
||||
echo implode(' ', $parts)."\n";
|
||||
|
|
|
@ -1,205 +0,0 @@
|
|||
#!/usr/bin/env php
|
||||
<?php
|
||||
|
||||
|
||||
$root = dirname(dirname(dirname(__FILE__)));
|
||||
require_once $root.'/scripts/__init_script__.php';
|
||||
|
||||
$args = new PhutilArgumentParser($argv);
|
||||
$args->setSynopsis(<<<EOSYNOPSIS
|
||||
**import_project_symbols.php** [__options__] __project_name__ < symbols
|
||||
|
||||
Import project symbols (symbols are read from stdin).
|
||||
EOSYNOPSIS
|
||||
);
|
||||
$args->parseStandardArguments();
|
||||
$args->parse(
|
||||
array(
|
||||
array(
|
||||
'name' => 'no-purge',
|
||||
'help' => 'Do not clear all symbols for this project before '.
|
||||
'uploading new symbols. Useful for incremental updating.',
|
||||
),
|
||||
array(
|
||||
'name' => 'ignore-errors',
|
||||
'help' => 'If a line can\'t be parsed, ignore that line and '.
|
||||
'continue instead of exiting.',
|
||||
),
|
||||
array(
|
||||
'name' => 'max-transaction',
|
||||
'param' => 'num-syms',
|
||||
'default' => '100000',
|
||||
'help' => 'Maximum number of symbols that should '.
|
||||
'be part of a single transaction',
|
||||
),
|
||||
array(
|
||||
'name' => 'more',
|
||||
'wildcard' => true,
|
||||
),
|
||||
));
|
||||
|
||||
$more = $args->getArg('more');
|
||||
if (count($more) !== 1) {
|
||||
$args->printHelpAndExit();
|
||||
}
|
||||
|
||||
$project_name = head($more);
|
||||
$project = id(new PhabricatorRepositoryArcanistProject())->loadOneWhere(
|
||||
'name = %s',
|
||||
$project_name);
|
||||
|
||||
if (!$project) {
|
||||
// TODO: Provide a less silly way to do this explicitly, or just do it right
|
||||
// here.
|
||||
echo "Project '{$project_name}' is unknown. Upload a diff to implicitly ".
|
||||
"create it.\n";
|
||||
exit(1);
|
||||
}
|
||||
|
||||
echo "Parsing input from stdin...\n";
|
||||
$input = file_get_contents('php://stdin');
|
||||
$input = trim($input);
|
||||
$input = explode("\n", $input);
|
||||
|
||||
|
||||
function commit_symbols ($syms, $project, $no_purge) {
|
||||
echo "Looking up path IDs...\n";
|
||||
$path_map =
|
||||
PhabricatorRepositoryCommitChangeParserWorker::lookupOrCreatePaths(
|
||||
ipull($syms, 'path'));
|
||||
|
||||
$symbol = new PhabricatorRepositorySymbol();
|
||||
$conn_w = $symbol->establishConnection('w');
|
||||
|
||||
echo "Preparing queries...\n";
|
||||
$sql = array();
|
||||
foreach ($syms as $dict) {
|
||||
$sql[] = qsprintf(
|
||||
$conn_w,
|
||||
'(%d, %s, %s, %s, %s, %d, %d)',
|
||||
$project->getID(),
|
||||
$dict['ctxt'],
|
||||
$dict['name'],
|
||||
$dict['type'],
|
||||
$dict['lang'],
|
||||
$dict['line'],
|
||||
$path_map[$dict['path']]);
|
||||
}
|
||||
|
||||
if (!$no_purge) {
|
||||
echo "Purging old syms...\n";
|
||||
queryfx($conn_w,
|
||||
'DELETE FROM %T WHERE arcanistProjectID = %d',
|
||||
$symbol->getTableName(),
|
||||
$project->getID());
|
||||
}
|
||||
|
||||
echo "Loading ".number_format(count($sql))." syms...\n";
|
||||
foreach (array_chunk($sql, 128) as $chunk) {
|
||||
queryfx($conn_w,
|
||||
'INSERT INTO %T
|
||||
(arcanistProjectID, symbolContext, symbolName, symbolType,
|
||||
symbolLanguage, lineNumber, pathID) VALUES %Q',
|
||||
$symbol->getTableName(),
|
||||
implode(', ', $chunk));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function check_string_value($value, $field_name, $line_no, $max_length) {
|
||||
if (strlen($value) > $max_length) {
|
||||
throw new Exception(
|
||||
"{$field_name} '{$value}' defined on line #{$line_no} is too long, ".
|
||||
"maximum {$field_name} length is {$max_length} characters.");
|
||||
}
|
||||
|
||||
if (!phutil_is_utf8_with_only_bmp_characters($value)) {
|
||||
throw new Exception(
|
||||
"{$field_name} '{$value}' defined on line #{$line_no} is not a valid ".
|
||||
"UTF-8 string, ".
|
||||
"it should contain only UTF-8 characters.");
|
||||
}
|
||||
}
|
||||
|
||||
$no_purge = $args->getArg('no-purge');
|
||||
$symbols = array();
|
||||
foreach ($input as $key => $line) {
|
||||
try {
|
||||
$line_no = $key + 1;
|
||||
$matches = null;
|
||||
$ok = preg_match(
|
||||
'/^((?P<context>[^ ]+)? )?(?P<name>[^ ]+) (?P<type>[^ ]+) '.
|
||||
'(?P<lang>[^ ]+) (?P<line>\d+) (?P<path>.*)$/',
|
||||
$line,
|
||||
$matches);
|
||||
if (!$ok) {
|
||||
throw new Exception(
|
||||
"Line #{$line_no} of input is invalid. Expected five or six ".
|
||||
"space-delimited fields: maybe symbol context, symbol name, symbol ".
|
||||
"type, symbol language, line number, path. ".
|
||||
"For example:\n\n".
|
||||
"idx function php 13 /path/to/some/file.php\n\n".
|
||||
"Actual line was:\n\n".
|
||||
"{$line}");
|
||||
}
|
||||
if (empty($matches['context'])) {
|
||||
$matches['context'] = '';
|
||||
}
|
||||
$context = $matches['context'];
|
||||
$name = $matches['name'];
|
||||
$type = $matches['type'];
|
||||
$lang = $matches['lang'];
|
||||
$line_number = $matches['line'];
|
||||
$path = $matches['path'];
|
||||
|
||||
check_string_value($context, 'Symbol context', $line_no, 128);
|
||||
check_string_value($name, 'Symbol name', $line_no, 128);
|
||||
check_string_value($type, 'Symbol type', $line_no, 12);
|
||||
check_string_value($lang, 'Symbol language', $line_no, 32);
|
||||
check_string_value($path, 'Path', $line_no, 512);
|
||||
|
||||
if (!strlen($path) || $path[0] != '/') {
|
||||
throw new Exception(
|
||||
"Path '{$path}' defined on line #{$line_no} is invalid. Paths should ".
|
||||
"begin with '/' and specify a path from the root of the project, like ".
|
||||
"'/src/utils/utils.php'.");
|
||||
}
|
||||
|
||||
$symbols[] = array(
|
||||
'ctxt' => $context,
|
||||
'name' => $name,
|
||||
'type' => $type,
|
||||
'lang' => $lang,
|
||||
'line' => $line_number,
|
||||
'path' => $path,
|
||||
);
|
||||
} catch (Exception $e) {
|
||||
if ($args->getArg('ignore-errors')) {
|
||||
continue;
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
if (count ($symbols) >= $args->getArg('max-transaction')) {
|
||||
try {
|
||||
echo "Committing {$args->getArg('max-transaction')} symbols....\n";
|
||||
commit_symbols($symbols, $project, $no_purge);
|
||||
$no_purge = true;
|
||||
unset($symbols);
|
||||
$symbols = array();
|
||||
} catch (Exception $e) {
|
||||
if ($args->getArg('ignore-errors')) {
|
||||
continue;
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (count($symbols)) {
|
||||
commit_symbols($symbols, $project, $no_purge);
|
||||
}
|
||||
|
||||
echo "Done.\n";
|
228
scripts/symbols/import_repository_symbols.php
Executable file
228
scripts/symbols/import_repository_symbols.php
Executable file
|
@ -0,0 +1,228 @@
|
|||
#!/usr/bin/env php
|
||||
<?php
|
||||
|
||||
$root = dirname(dirname(dirname(__FILE__)));
|
||||
require_once $root.'/scripts/__init_script__.php';
|
||||
|
||||
$args = new PhutilArgumentParser($argv);
|
||||
$args->setSynopsis(<<<EOSYNOPSIS
|
||||
**import_repository_symbols.php** [__options__] __callsign__ < symbols
|
||||
|
||||
Import repository symbols (symbols are read from stdin).
|
||||
EOSYNOPSIS
|
||||
);
|
||||
$args->parseStandardArguments();
|
||||
$args->parse(
|
||||
array(
|
||||
array(
|
||||
'name' => 'no-purge',
|
||||
'help' => pht(
|
||||
'Do not clear all symbols for this repository before '.
|
||||
'uploading new symbols. Useful for incremental updating.'),
|
||||
),
|
||||
array(
|
||||
'name' => 'ignore-errors',
|
||||
'help' => pht(
|
||||
"If a line can't be parsed, ignore that line and ".
|
||||
"continue instead of exiting."),
|
||||
),
|
||||
array(
|
||||
'name' => 'max-transaction',
|
||||
'param' => 'num-syms',
|
||||
'default' => '100000',
|
||||
'help' => pht(
|
||||
'Maximum number of symbols that should '.
|
||||
'be part of a single transaction.'),
|
||||
),
|
||||
array(
|
||||
'name' => 'more',
|
||||
'wildcard' => true,
|
||||
),
|
||||
));
|
||||
|
||||
$more = $args->getArg('more');
|
||||
if (count($more) !== 1) {
|
||||
$args->printHelpAndExit();
|
||||
}
|
||||
|
||||
$callsign = head($more);
|
||||
$repository = id(new PhabricatorRepository())->loadOneWhere(
|
||||
'callsign = %s',
|
||||
$callsign);
|
||||
|
||||
if (!$repository) {
|
||||
echo pht("Repository '%s' does not exist.", $callsign);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (!function_exists('posix_isatty') || posix_isatty(STDIN)) {
|
||||
echo pht('Parsing input from stdin...'), "\n";
|
||||
}
|
||||
|
||||
$input = file_get_contents('php://stdin');
|
||||
$input = trim($input);
|
||||
$input = explode("\n", $input);
|
||||
|
||||
|
||||
function commit_symbols(
|
||||
array $symbols,
|
||||
PhabricatorRepository $repository,
|
||||
$no_purge) {
|
||||
|
||||
echo pht('Looking up path IDs...'), "\n";
|
||||
$path_map =
|
||||
PhabricatorRepositoryCommitChangeParserWorker::lookupOrCreatePaths(
|
||||
ipull($symbols, 'path'));
|
||||
|
||||
$symbol = new PhabricatorRepositorySymbol();
|
||||
$conn_w = $symbol->establishConnection('w');
|
||||
|
||||
echo pht('Preparing queries...'), "\n";
|
||||
$sql = array();
|
||||
foreach ($symbols as $dict) {
|
||||
$sql[] = qsprintf(
|
||||
$conn_w,
|
||||
'(%s, %s, %s, %s, %s, %d, %d)',
|
||||
$repository->getPHID(),
|
||||
$dict['ctxt'],
|
||||
$dict['name'],
|
||||
$dict['type'],
|
||||
$dict['lang'],
|
||||
$dict['line'],
|
||||
$path_map[$dict['path']]);
|
||||
}
|
||||
|
||||
if (!$no_purge) {
|
||||
echo pht('Purging old symbols...'), "\n";
|
||||
queryfx(
|
||||
$conn_w,
|
||||
'DELETE FROM %T WHERE repositoryPHID = %s',
|
||||
$symbol->getTableName(),
|
||||
$repository->getPHID());
|
||||
}
|
||||
|
||||
echo pht('Loading %s symbols...', new PhutilNumber(count($sql))), "\n";
|
||||
foreach (array_chunk($sql, 128) as $chunk) {
|
||||
queryfx(
|
||||
$conn_w,
|
||||
'INSERT INTO %T
|
||||
(repositoryPHID, symbolContext, symbolName, symbolType,
|
||||
symbolLanguage, lineNumber, pathID) VALUES %Q',
|
||||
$symbol->getTableName(),
|
||||
implode(', ', $chunk));
|
||||
}
|
||||
}
|
||||
|
||||
function check_string_value($value, $field_name, $line_no, $max_length) {
|
||||
if (strlen($value) > $max_length) {
|
||||
throw new Exception(
|
||||
pht(
|
||||
"%s '%s' defined on line #%d is too long, ".
|
||||
"maximum %s length is %d characters.",
|
||||
$field_name,
|
||||
$value,
|
||||
$line_no,
|
||||
$field_name,
|
||||
$max_length));
|
||||
}
|
||||
|
||||
if (!phutil_is_utf8_with_only_bmp_characters($value)) {
|
||||
throw new Exception(
|
||||
pht(
|
||||
"%s '%s' defined on line #%d is not a valid ".
|
||||
"UTF-8 string, it should contain only UTF-8 characters.",
|
||||
$field_name,
|
||||
$value,
|
||||
$line_no));
|
||||
}
|
||||
}
|
||||
|
||||
$no_purge = $args->getArg('no-purge');
|
||||
$symbols = array();
|
||||
foreach ($input as $key => $line) {
|
||||
try {
|
||||
$line_no = $key + 1;
|
||||
$matches = null;
|
||||
$ok = preg_match(
|
||||
'/^((?P<context>[^ ]+)? )?(?P<name>[^ ]+) (?P<type>[^ ]+) '.
|
||||
'(?P<lang>[^ ]+) (?P<line>\d+) (?P<path>.*)$/',
|
||||
$line,
|
||||
$matches);
|
||||
if (!$ok) {
|
||||
throw new Exception(
|
||||
pht(
|
||||
"Line #%d of input is invalid. Expected five or six space-delimited ".
|
||||
"fields: maybe symbol context, symbol name, symbol type, symbol ".
|
||||
"language, line number, path. For example:\n\n%s\n\n".
|
||||
"Actual line was:\n\n%s",
|
||||
$line_no,
|
||||
'idx function php 13 /path/to/some/file.php',
|
||||
$line));
|
||||
}
|
||||
if (empty($matches['context'])) {
|
||||
$matches['context'] = '';
|
||||
}
|
||||
$context = $matches['context'];
|
||||
$name = $matches['name'];
|
||||
$type = $matches['type'];
|
||||
$lang = $matches['lang'];
|
||||
$line_number = $matches['line'];
|
||||
$path = $matches['path'];
|
||||
|
||||
check_string_value($context, 'Symbol context', $line_no, 128);
|
||||
check_string_value($name, 'Symbol name', $line_no, 128);
|
||||
check_string_value($type, 'Symbol type', $line_no, 12);
|
||||
check_string_value($lang, 'Symbol language', $line_no, 32);
|
||||
check_string_value($path, 'Path', $line_no, 512);
|
||||
|
||||
if (!strlen($path) || $path[0] != '/') {
|
||||
throw new Exception(
|
||||
pht(
|
||||
"Path '%s' defined on line #%d is invalid. Paths should begin with ".
|
||||
"'%s' and specify a path from the root of the project, like '%s'.",
|
||||
$path,
|
||||
$line_no,
|
||||
'/',
|
||||
'/src/utils/utils.php'));
|
||||
}
|
||||
|
||||
$symbols[] = array(
|
||||
'ctxt' => $context,
|
||||
'name' => $name,
|
||||
'type' => $type,
|
||||
'lang' => $lang,
|
||||
'line' => $line_number,
|
||||
'path' => $path,
|
||||
);
|
||||
} catch (Exception $e) {
|
||||
if ($args->getArg('ignore-errors')) {
|
||||
continue;
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
if (count ($symbols) >= $args->getArg('max-transaction')) {
|
||||
try {
|
||||
echo pht(
|
||||
"Committing %s symbols...\n",
|
||||
new PhutilNumber($args->getArg('max-transaction')));
|
||||
commit_symbols($symbols, $repository, $no_purge);
|
||||
$no_purge = true;
|
||||
unset($symbols);
|
||||
$symbols = array();
|
||||
} catch (Exception $e) {
|
||||
if ($args->getArg('ignore-errors')) {
|
||||
continue;
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (count($symbols)) {
|
||||
commit_symbols($symbols, $repository, $no_purge);
|
||||
}
|
||||
|
||||
echo pht('Done.'), "\n";
|
|
@ -8,7 +8,7 @@ final class DiffusionFindSymbolsConduitAPIMethod
|
|||
}
|
||||
|
||||
public function getMethodDescription() {
|
||||
return 'Retrieve Diffusion symbol information.';
|
||||
return pht('Retrieve Diffusion symbol information.');
|
||||
}
|
||||
|
||||
protected function defineParamTypes() {
|
||||
|
@ -51,7 +51,6 @@ final class DiffusionFindSymbolsConduitAPIMethod
|
|||
}
|
||||
|
||||
$query->needPaths(true);
|
||||
$query->needArcanistProjects(true);
|
||||
$query->needRepositories(true);
|
||||
|
||||
$results = $query->execute();
|
||||
|
|
|
@ -24,25 +24,25 @@ final class DiffusionSymbolController extends DiffusionController {
|
|||
$query->setLanguage($request->getStr('lang'));
|
||||
}
|
||||
|
||||
if ($request->getStr('projects')) {
|
||||
$phids = $request->getStr('projects');
|
||||
if ($request->getStr('repositories')) {
|
||||
$phids = $request->getStr('repositories');
|
||||
$phids = explode(',', $phids);
|
||||
$phids = array_filter($phids);
|
||||
|
||||
if ($phids) {
|
||||
$projects = id(new PhabricatorRepositoryArcanistProject())
|
||||
->loadAllWhere(
|
||||
'phid IN (%Ls)',
|
||||
$phids);
|
||||
$projects = mpull($projects, 'getID');
|
||||
if ($projects) {
|
||||
$query->setProjectIDs($projects);
|
||||
$repos = id(new PhabricatorRepositoryQuery())
|
||||
->setViewer($request->getUser())
|
||||
->withPHIDs($phids)
|
||||
->execute();
|
||||
|
||||
$repos = mpull($repos, 'getPHID');
|
||||
if ($repos) {
|
||||
$query->withRepositoryPHIDs($repos);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$query->needPaths(true);
|
||||
$query->needArcanistProjects(true);
|
||||
$query->needRepositories(true);
|
||||
|
||||
$symbols = $query->execute();
|
||||
|
@ -73,13 +73,6 @@ final class DiffusionSymbolController extends DiffusionController {
|
|||
|
||||
$rows = array();
|
||||
foreach ($symbols as $symbol) {
|
||||
$project = $symbol->getArcanistProject();
|
||||
if ($project) {
|
||||
$project_name = $project->getName();
|
||||
} else {
|
||||
$project_name = '-';
|
||||
}
|
||||
|
||||
$file = $symbol->getPath();
|
||||
$line = $symbol->getLineNumber();
|
||||
|
||||
|
@ -110,7 +103,7 @@ final class DiffusionSymbolController extends DiffusionController {
|
|||
$symbol->getSymbolContext(),
|
||||
$symbol->getSymbolName(),
|
||||
$symbol->getSymbolLanguage(),
|
||||
$project_name,
|
||||
$repo->getMonogram(),
|
||||
$location,
|
||||
);
|
||||
}
|
||||
|
@ -122,7 +115,7 @@ final class DiffusionSymbolController extends DiffusionController {
|
|||
pht('Context'),
|
||||
pht('Name'),
|
||||
pht('Language'),
|
||||
pht('Project'),
|
||||
pht('Repository'),
|
||||
pht('File'),
|
||||
));
|
||||
$table->setColumnClasses(
|
||||
|
|
|
@ -16,12 +16,11 @@ final class DiffusionSymbolQuery extends PhabricatorOffsetPagedQuery {
|
|||
private $namePrefix;
|
||||
private $name;
|
||||
|
||||
private $projectIDs;
|
||||
private $repositoryPHIDs;
|
||||
private $language;
|
||||
private $type;
|
||||
|
||||
private $needPaths;
|
||||
private $needArcanistProject;
|
||||
private $needRepositories;
|
||||
|
||||
|
||||
|
@ -72,8 +71,8 @@ final class DiffusionSymbolQuery extends PhabricatorOffsetPagedQuery {
|
|||
/**
|
||||
* @task config
|
||||
*/
|
||||
public function setProjectIDs(array $project_ids) {
|
||||
$this->projectIDs = $project_ids;
|
||||
public function withRepositoryPHIDs(array $repository_phids) {
|
||||
$this->repositoryPHIDs = $repository_phids;
|
||||
return $this;
|
||||
}
|
||||
|
||||
|
@ -105,15 +104,6 @@ final class DiffusionSymbolQuery extends PhabricatorOffsetPagedQuery {
|
|||
}
|
||||
|
||||
|
||||
/**
|
||||
* @task config
|
||||
*/
|
||||
public function needArcanistProjects($need_arcanist_projects) {
|
||||
$this->needArcanistProjects = $need_arcanist_projects;
|
||||
return $this;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @task config
|
||||
*/
|
||||
|
@ -132,10 +122,10 @@ final class DiffusionSymbolQuery extends PhabricatorOffsetPagedQuery {
|
|||
public function execute() {
|
||||
if ($this->name && $this->namePrefix) {
|
||||
throw new Exception(
|
||||
'You can not set both a name and a name prefix!');
|
||||
pht('You can not set both a name and a name prefix!'));
|
||||
} else if (!$this->name && !$this->namePrefix) {
|
||||
throw new Exception(
|
||||
'You must set a name or a name prefix!');
|
||||
pht('You must set a name or a name prefix!'));
|
||||
}
|
||||
|
||||
$symbol = new PhabricatorRepositorySymbol();
|
||||
|
@ -155,9 +145,6 @@ final class DiffusionSymbolQuery extends PhabricatorOffsetPagedQuery {
|
|||
if ($this->needPaths) {
|
||||
$this->loadPaths($symbols);
|
||||
}
|
||||
if ($this->needArcanistProjects || $this->needRepositories) {
|
||||
$this->loadArcanistProjects($symbols);
|
||||
}
|
||||
if ($this->needRepositories) {
|
||||
$this->loadRepositories($symbols);
|
||||
}
|
||||
|
@ -208,11 +195,11 @@ final class DiffusionSymbolQuery extends PhabricatorOffsetPagedQuery {
|
|||
$this->namePrefix);
|
||||
}
|
||||
|
||||
if ($this->projectIDs) {
|
||||
if ($this->repositoryPHIDs) {
|
||||
$where[] = qsprintf(
|
||||
$conn_r,
|
||||
'arcanistProjectID IN (%Ld)',
|
||||
$this->projectIDs);
|
||||
'repositoryPHID IN (%Ls)',
|
||||
$this->repositoryPHIDs);
|
||||
}
|
||||
|
||||
if ($this->language) {
|
||||
|
@ -250,49 +237,21 @@ final class DiffusionSymbolQuery extends PhabricatorOffsetPagedQuery {
|
|||
}
|
||||
|
||||
|
||||
/**
|
||||
* @task internal
|
||||
*/
|
||||
private function loadArcanistProjects(array $symbols) {
|
||||
assert_instances_of($symbols, 'PhabricatorRepositorySymbol');
|
||||
$projects = id(new PhabricatorRepositoryArcanistProject())->loadAllWhere(
|
||||
'id IN (%Ld)',
|
||||
mpull($symbols, 'getArcanistProjectID'));
|
||||
foreach ($symbols as $symbol) {
|
||||
$project = idx($projects, $symbol->getArcanistProjectID());
|
||||
$symbol->attachArcanistProject($project);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @task internal
|
||||
*/
|
||||
private function loadRepositories(array $symbols) {
|
||||
assert_instances_of($symbols, 'PhabricatorRepositorySymbol');
|
||||
|
||||
$projects = mpull($symbols, 'getArcanistProject');
|
||||
$projects = array_filter($projects);
|
||||
|
||||
$repo_ids = mpull($projects, 'getRepositoryID');
|
||||
$repo_ids = array_filter($repo_ids);
|
||||
|
||||
if ($repo_ids) {
|
||||
$repos = id(new PhabricatorRepositoryQuery())
|
||||
->setViewer($this->getViewer())
|
||||
->withIDs($repo_ids)
|
||||
->execute();
|
||||
} else {
|
||||
$repos = array();
|
||||
}
|
||||
$repos = id(new PhabricatorRepositoryQuery())
|
||||
->setViewer($this->viewer)
|
||||
->withPHIDs(mpull($symbols, 'getRepositoryPHID'))
|
||||
->execute();
|
||||
$repos = mpull($repos, null, 'getPHID');
|
||||
|
||||
foreach ($symbols as $symbol) {
|
||||
$proj = $symbol->getArcanistProject();
|
||||
if ($proj) {
|
||||
$symbol->attachRepository(idx($repos, $proj->getRepositoryID()));
|
||||
} else {
|
||||
$symbol->attachRepository(null);
|
||||
}
|
||||
$repository = idx($repos, $symbol->getRepositoryPHID());
|
||||
$symbol->attachRepository($repository);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -32,7 +32,6 @@ final class DiffusionSymbolDatasource
|
|||
->setViewer($viewer)
|
||||
->setNamePrefix($raw_query)
|
||||
->setLimit(15)
|
||||
->needArcanistProjects(true)
|
||||
->needRepositories(true)
|
||||
->needPaths(true)
|
||||
->execute();
|
||||
|
@ -40,14 +39,14 @@ final class DiffusionSymbolDatasource
|
|||
$lang = $symbol->getSymbolLanguage();
|
||||
$name = $symbol->getSymbolName();
|
||||
$type = $symbol->getSymbolType();
|
||||
$proj = $symbol->getArcanistProject()->getName();
|
||||
$repo = $symbol->getRepository()->getName();
|
||||
|
||||
$results[] = id(new PhabricatorTypeaheadResult())
|
||||
->setName($name)
|
||||
->setURI($symbol->getURI())
|
||||
->setPHID(md5($symbol->getURI())) // Just needs to be unique.
|
||||
->setDisplayName($name)
|
||||
->setDisplayType(strtoupper($lang).' '.ucwords($type).' ('.$proj.')')
|
||||
->setDisplayType(strtoupper($lang).' '.ucwords($type).' ('.$repo.')')
|
||||
->setPriorityType('symb');
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1163,10 +1163,15 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
|
|||
$projects = id(new PhabricatorRepositoryArcanistProject())
|
||||
->loadAllWhere('repositoryID = %d', $this->getID());
|
||||
foreach ($projects as $project) {
|
||||
// note each project deletes its PhabricatorRepositorySymbols
|
||||
$project->delete();
|
||||
}
|
||||
|
||||
queryfx(
|
||||
$this->establishConnection('w'),
|
||||
'DELETE FROM %T WHERE repositoryPHID = %s',
|
||||
id(new PhabricatorRepositorySymbol())->getTableName(),
|
||||
$this->getPHID());
|
||||
|
||||
$commits = id(new PhabricatorRepositoryCommit())
|
||||
->loadAllWhere('repositoryID = %d', $this->getID());
|
||||
foreach ($commits as $commit) {
|
||||
|
|
|
@ -45,20 +45,6 @@ final class PhabricatorRepositoryArcanistProject
|
|||
PhabricatorRepositoryArcanistProjectPHIDType::TYPECONST);
|
||||
}
|
||||
|
||||
public function delete() {
|
||||
$this->openTransaction();
|
||||
|
||||
queryfx(
|
||||
$this->establishConnection('w'),
|
||||
'DELETE FROM %T WHERE arcanistProjectID = %d',
|
||||
id(new PhabricatorRepositorySymbol())->getTableName(),
|
||||
$this->getID());
|
||||
|
||||
$result = parent::delete();
|
||||
$this->saveTransaction();
|
||||
return $result;
|
||||
}
|
||||
|
||||
public function getRepository() {
|
||||
return $this->assertAttached($this->repository);
|
||||
}
|
||||
|
|
|
@ -8,7 +8,7 @@
|
|||
*/
|
||||
final class PhabricatorRepositorySymbol extends PhabricatorRepositoryDAO {
|
||||
|
||||
protected $arcanistProjectID;
|
||||
protected $repositoryPHID;
|
||||
protected $symbolContext;
|
||||
protected $symbolName;
|
||||
protected $symbolType;
|
||||
|
@ -17,12 +17,10 @@ final class PhabricatorRepositorySymbol extends PhabricatorRepositoryDAO {
|
|||
protected $lineNumber;
|
||||
|
||||
private $path = self::ATTACHABLE;
|
||||
private $arcanistProject = self::ATTACHABLE;
|
||||
private $repository = self::ATTACHABLE;
|
||||
|
||||
protected function getConfiguration() {
|
||||
return array(
|
||||
self::CONFIG_IDS => self::IDS_MANUAL,
|
||||
self::CONFIG_TIMESTAMPS => false,
|
||||
self::CONFIG_COLUMN_SCHEMA => array(
|
||||
'id' => null,
|
||||
|
@ -42,13 +40,6 @@ final class PhabricatorRepositorySymbol extends PhabricatorRepositoryDAO {
|
|||
}
|
||||
|
||||
public function getURI() {
|
||||
if (!$this->repository) {
|
||||
// This symbol is in the index, but we don't know which Repository it's
|
||||
// part of. Usually this means the Arcanist Project hasn't been linked
|
||||
// to a Repository. We can't generate a URI, so just fail.
|
||||
return null;
|
||||
}
|
||||
|
||||
$request = DiffusionRequest::newFromDictionary(
|
||||
array(
|
||||
'user' => PhabricatorUser::getOmnipotentUser(),
|
||||
|
@ -75,18 +66,9 @@ final class PhabricatorRepositorySymbol extends PhabricatorRepositoryDAO {
|
|||
return $this->assertAttached($this->repository);
|
||||
}
|
||||
|
||||
public function attachRepository($repository) {
|
||||
public function attachRepository(PhabricatorRepository $repository) {
|
||||
$this->repository = $repository;
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getArcanistProject() {
|
||||
return $this->assertAttached($this->arcanistProject);
|
||||
}
|
||||
|
||||
public function attachArcanistProject($project) {
|
||||
$this->arcanistProject = $project;
|
||||
return $this;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue