Move symbols to be repository-based

Summary: Fixes T7220. Ref T7977. Changes symbols from being bound to an Arcanist project to being bound to a repository.

Test Plan:
- Added symbols and then applied migrations, symbols seemed to be migrated successfully.
- Tested the `/diffusion/symbol/$SYMBOL_NAME` endpoint.
- Tested the `/diffusion/symbol/$SYMBOL_NAME` endpoint with the `?repositories=$REPOSITORY_PHID` parameter.

Reviewers: #blessed_reviewers, epriestley

Reviewed By: #blessed_reviewers, epriestley

Subscribers: avivey, Korvin, epriestley

Maniphest Tasks: T7977, T7220

Differential Revision: https://secure.phabricator.com/D12608

resources/sql/autopatches/20150503.repositorysymbols.1.sql

@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_repository.repository_symbol
+  ADD repositoryPHID varbinary(64) NOT NULL AFTER arcanistProjectID;

resources/sql/autopatches/20150503.repositorysymbols.2.php

@@ -0,0 +1,26 @@
+<?php
+
+$projects = id(new PhabricatorRepositoryArcanistProjectQuery())
+  ->setViewer(PhabricatorUser::getOmnipotentUser())
+  ->needRepositories(true)
+  ->execute();
+
+$table = new PhabricatorRepositorySymbol();
+$conn_w = $table->establishConnection('w');
+
+foreach ($projects as $project) {
+  $repo = $project->getRepository();
+
+  if (!$repo) {
+    continue;
+  }
+
+  echo pht("Migrating symbols for '%s' project...\n", $project->getName());
+
+  queryfx(
+    $conn_w,
+    'UPDATE %T SET repositoryPHID = %s WHERE arcanistProjectID = %d',
+    $table->getTableName(),
+    $repo->getPHID(),
+    $project->getID());
+}

resources/sql/autopatches/20150503.repositorysymbols.3.sql

@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_repository.repository_symbol
+  DROP COLUMN arcanistProjectID;

scripts/symbols/clear_project_symbols.php

@@ -1,32 +0,0 @@
-#!/usr/bin/env php
-<?php
-
-$root = dirname(dirname(dirname(__FILE__)));
-require_once $root.'/scripts/__init_script__.php';
-
-$project = id(new PhabricatorRepositoryArcanistProject())->loadOneWhere(
-  'name = %s', $argv[1]);
-if (!$project) {
-  throw new Exception('No such arcanist project.');
-}
-
-$input = file_get_contents('php://stdin');
-$normalized = array();
-foreach (explode("\n", trim($input)) as $path) {
-  // emulate the behavior of the symbol generation scripts
-  $normalized[] = '/'.ltrim($path, './');
-}
-$paths = PhabricatorRepositoryCommitChangeParserWorker::lookupOrCreatePaths(
-  $normalized);
-
-$symbol = new PhabricatorRepositorySymbol();
-$conn_w = $symbol->establishConnection('w');
-
-foreach (array_chunk(array_values($paths), 128) as $chunk) {
-  queryfx(
-    $conn_w,
-    'DELETE FROM %T WHERE arcanistProjectID = %d AND pathID IN (%Ld)',
-    $symbol->getTableName(),
-    $project->getID(),
-    $chunk);
-}

scripts/symbols/clear_repository_symbols.php

@@ -0,0 +1,58 @@
+#!/usr/bin/env php
+<?php
+
+$root = dirname(dirname(dirname(__FILE__)));
+require_once $root.'/scripts/__init_script__.php';
+
+$args = new PhutilArgumentParser($argv);
+$args->setSynopsis(<<<EOSYNOPSIS
+**clear_repository_symbols.php** [__options__] __callsign__
+
+  Clear repository symbols.
+EOSYNOPSIS
+  );
+$args->parseStandardArguments();
+$args->parse(
+  array(
+    array(
+      'name'      => 'callsign',
+      'wildcard'  => true,
+    ),
+  ));
+
+$callsigns = $args->getArg('callsign');
+if (count($callsigns) !== 1) {
+  $args->printHelpAndExit();
+}
+
+$callsign = head($callsigns);
+$repository = id(new PhabricatorRepositoryQuery())
+  ->setViewer(PhabricatorUser::getOmnipotentUser())
+  ->withCallsigns($callsigns)
+  ->executeOne();
+
+if (!$repository) {
+  echo pht("Repository '%s' does not exist.", $callsign);
+  exit(1);
+}
+
+$input = file_get_contents('php://stdin');
+$normalized = array();
+foreach (explode("\n", trim($input)) as $path) {
+  // Emulate the behavior of the symbol generation scripts.
+  $normalized[] = '/'.ltrim($path, './');
+}
+$paths = PhabricatorRepositoryCommitChangeParserWorker::lookupOrCreatePaths(
+  $normalized);
+
+$symbol = new PhabricatorRepositorySymbol();
+$conn_w = $symbol->establishConnection('w');
+
+foreach (array_chunk(array_values($paths), 128) as $chunk) {
+  queryfx(
+    $conn_w,
+    'DELETE FROM %T WHERE repositoryPHID = %s AND pathID IN (%Ld)',
+    $symbol->getTableName(),
+    $repository->getPHID(),
+    $chunk);
+}

scripts/symbols/generate_ctags_symbols.php

@@ -4,79 +4,94 @@
 $root = dirname(dirname(dirname(__FILE__)));
 require_once $root.'/scripts/__init_script__.php';
 
+$args = new PhutilArgumentParser($argv);
+$args->setSynopsis(<<<EOSYNOPSIS
+**generate_ctags_symbols.php** [__options__]
+
+  Generate repository symbols using Exuberant Ctags. Paths are read from stdin.
+EOSYNOPSIS
+  );
+$args->parseStandardArguments();
+
 if (ctags_check_executable() == false) {
   echo phutil_console_format(
-    "Could not find Exuberant ctags. Make sure it is installed and\n".
+    "%s\n\n%s\n",
-    "available in executable path.\n\n".
+    pht(
-    "Exuberant ctags project page: http://ctags.sourceforge.net/\n");
+      'Could not find Exuberant Ctags. Make sure it is installed and '.
+      'available in executable path.'),
+    pht(
+      'Exuberant Ctags project page: %s',
+      'http://ctags.sourceforge.net/'));
   exit(1);
 }
 
-if ($argc !== 1 || posix_isatty(STDIN)) {
+if (posix_isatty(STDIN)) {
   echo phutil_console_format(
-    "usage: find . -type f -name '*.py' | ./generate_ctags_symbols.php\n");
+    "%s\n",
+    pht(
+      'Usage: %s',
+      "find . -type f -name '*.py' | ./generate_ctags_symbols.php"));
   exit(1);
 }
 
 $input = file_get_contents('php://stdin');
-$input = trim($input);
-$input = explode("\n", $input);
-
 $data = array();
 $futures = array();
 
-foreach ($input as $file) {
+foreach (explode("\n", trim($input)) as $file) {
   $file = Filesystem::readablePath($file);
   $futures[$file] = ctags_get_parser_future($file);
 }
 
-$futures = id(new FutureIterator($futures))
+$futures = new FutureIterator($futures);
-  ->limit(8);
+foreach ($futures->limit(8) as $file => $future) {
-foreach ($futures as $file => $future) {
   $tags = $future->resolve();
   $tags = explode("\n", $tags[1]);
 
   foreach ($tags as $tag) {
     $parts = explode(';', $tag);
-    // skip lines that we can not parse
+
+    // Skip lines that we can not parse.
     if (count($parts) < 2) {
       continue;
     }
 
-    // split ctags information
+    // Split ctags information.
     $tag_info = explode("\t", $parts[0]);
-    // split exuberant ctags "extension fields" (additional information)
+
+    // Split exuberant ctags "extension fields" (additional information).
     $parts[1] = trim($parts[1], "\t \"");
     $extension_fields = explode("\t", $parts[1]);
 
-    // skip lines that we can not parse
+    // Skip lines that we can not parse.
     if (count($tag_info) < 3 || count($extension_fields) < 2) {
       continue;
     }
 
-    // default $context to empty
+    // Default context to empty.
     $extension_fields[] = '';
     list($token, $file_path, $line_num) = $tag_info;
     list($type, $language, $context) = $extension_fields;
 
-    // skip lines with tokens containing a space
+    // Skip lines with tokens containing a space.
     if (strpos($token, ' ') !== false) {
       continue;
     }
 
-    // strip "language:"
+    // Strip "language:"
     $language = substr($language, 9);
 
     // To keep consistent with "Separate with commas, for example: php, py"
     // in Arcanist Project edit form.
     $language = str_ireplace('python', 'py', $language);
 
-    // also, "normalize" c++ and c#
+    // Also, "normalize" C++ and C#.
     $language = str_ireplace('c++', 'cpp', $language);
     $language = str_ireplace('c#', 'cs', $language);
 
-    // Ruby has "singleton method", for example
+    // Ruby has "singleton method", for example.
     $type = substr(str_replace(' ', '_', $type), 0, 12);
+
     // class:foo, struct:foo, union:foo, enum:foo, ...
     $context = last(explode(':', $context, 2));
 
@@ -89,25 +104,18 @@ foreach ($futures as $file => $future) {
   }
 }
 
-function ctags_get_parser_future($file_path) {
+function ctags_get_parser_future($path) {
-  $future = new ExecFuture('ctags -n --fields=Kls -o - %s',
+  $future = new ExecFuture('ctags -n --fields=Kls -o - %s', $path);
-                           $file_path);
   return $future;
 }
 
 function ctags_check_executable() {
-  $future = new ExecFuture('ctags --version');
+  $result = exec_manual('ctags --version');
-  $result = $future->resolve();
+  return !empty($result[1]);
-
-  if (empty($result[1])) {
-    return false;
-  }
-
-  return true;
 }
 
 function print_symbol($file, $line_num, $type, $token, $context, $language) {
-  // get rid of relative path
+  // Get rid of relative path.
   $file = explode('/', $file);
   if ($file[0] == '.' || $file[0] == '..') {
     array_shift($file);

scripts/symbols/generate_php_symbols.php

@@ -4,28 +4,36 @@
 $root = dirname(dirname(dirname(__FILE__)));
 require_once $root.'/scripts/__init_script__.php';
 
-if ($argc !== 1 || posix_isatty(STDIN)) {
+$args = new PhutilArgumentParser($argv);
+$args->setSynopsis(<<<EOSYNOPSIS
+**generate_php_symbols.php** [__options__]
+
+  Generate repository symbols using XHPAST. Paths are read from stdin.
+EOSYNOPSIS
+  );
+$args->parseStandardArguments();
+
+if (posix_isatty(STDIN)) {
   echo phutil_console_format(
-    "usage: find . -type f -name '*.php' | ./generate_php_symbols.php\n");
+    "%s\n",
+    pht(
+      'Usage: %s',
+      "find . -type f -name '*.php' | ./generate_php_symbols.php"));
   exit(1);
 }
 
 $input = file_get_contents('php://stdin');
-$input = trim($input);
-$input = explode("\n", $input);
-
 $data = array();
 $futures = array();
 
-foreach ($input as $file) {
+foreach (explode("\n", trim($input)) as $file) {
   $file = Filesystem::readablePath($file);
   $data[$file] = Filesystem::readFile($file);
   $futures[$file] = PhutilXHPASTBinary::getParserFuture($data[$file]);
 }
 
-$futures = id(new FutureIterator($futures))
+$futures = new FutureIterator($futures);
-  ->limit(8);
+foreach ($futures->limit(8) as $file => $future) {
-foreach ($futures as $file => $future) {
   $tree = XHPASTTree::newFromDataAndResolvedExecFuture(
     $data[$file],
     $future->resolve());
@@ -36,7 +44,7 @@ foreach ($futures as $file => $future) {
   $functions = $root->selectDescendantsOfType('n_FUNCTION_DECLARATION');
   foreach ($functions as $function) {
     $name = $function->getChildByIndex(2);
-    // Skip anonymous functions
+    // Skip anonymous functions.
     if (!$name->getConcreteString()) {
       continue;
     }
@@ -67,8 +75,8 @@ foreach ($futures as $file => $future) {
   }
 
   foreach ($scopes as $scope) {
-    // this prints duplicate symbols in the case of nested classes
+    // This prints duplicate symbols in the case of nested classes.
-    // luckily, PHP doesn't allow those
+    // Luckily, PHP doesn't allow those.
     list($class, $class_name) = $scope;
 
     $consts = $class->selectDescendantsOfType(
@@ -100,15 +108,15 @@ foreach ($futures as $file => $future) {
   }
 }
 
-function print_symbol($file, $type, $token, $context = null) {
+function print_symbol($file, $type, XHPASTNode $node, $context = null) {
   $parts = array(
     $context ? $context->getConcreteString() : '',
-    // variable tokens are `$name`, not just `name`, so strip the $ off of
+    // Variable tokens are `$name`, not just `name`, so strip the "$"" off of
     // class field names
-    ltrim($token->getConcreteString(), '$'),
+    ltrim($node->getConcreteString(), '$'),
     $type,
     'php',
-    $token->getLineNumber(),
+    $node->getLineNumber(),
     '/'.ltrim($file, './'),
   );
   echo implode(' ', $parts)."\n";

scripts/symbols/import_project_symbols.php

@@ -1,205 +0,0 @@
-#!/usr/bin/env php
-<?php
-
-
-$root = dirname(dirname(dirname(__FILE__)));
-require_once $root.'/scripts/__init_script__.php';
-
-$args = new PhutilArgumentParser($argv);
-$args->setSynopsis(<<<EOSYNOPSIS
-**import_project_symbols.php** [__options__] __project_name__ < symbols
-
-  Import project symbols (symbols are read from stdin).
-EOSYNOPSIS
-  );
-$args->parseStandardArguments();
-$args->parse(
-  array(
-    array(
-      'name'      => 'no-purge',
-      'help'      => 'Do not clear all symbols for this project before '.
-                     'uploading new symbols. Useful for incremental updating.',
-    ),
-    array(
-      'name'      => 'ignore-errors',
-      'help'      => 'If a line can\'t be parsed, ignore that line and '.
-                     'continue instead of exiting.',
-    ),
-    array(
-      'name'      => 'max-transaction',
-      'param'     => 'num-syms',
-      'default'  => '100000',
-      'help'      => 'Maximum number of symbols that should '.
-                     'be part of a single transaction',
-    ),
-    array(
-      'name'      => 'more',
-      'wildcard'  => true,
-    ),
-  ));
-
-$more = $args->getArg('more');
-if (count($more) !== 1) {
-  $args->printHelpAndExit();
-}
-
-$project_name = head($more);
-$project = id(new PhabricatorRepositoryArcanistProject())->loadOneWhere(
-  'name = %s',
-  $project_name);
-
-if (!$project) {
-  // TODO: Provide a less silly way to do this explicitly, or just do it right
-  // here.
-  echo "Project '{$project_name}' is unknown. Upload a diff to implicitly ".
-       "create it.\n";
-  exit(1);
-}
-
-echo "Parsing input from stdin...\n";
-$input = file_get_contents('php://stdin');
-$input = trim($input);
-$input = explode("\n", $input);
-
-
-function commit_symbols ($syms, $project, $no_purge) {
-  echo "Looking up path IDs...\n";
-  $path_map =
-    PhabricatorRepositoryCommitChangeParserWorker::lookupOrCreatePaths(
-           ipull($syms, 'path'));
-
-  $symbol = new PhabricatorRepositorySymbol();
-  $conn_w = $symbol->establishConnection('w');
-
-  echo "Preparing queries...\n";
-  $sql = array();
-  foreach ($syms as $dict) {
-    $sql[] = qsprintf(
-                      $conn_w,
-                      '(%d, %s, %s, %s, %s, %d, %d)',
-                      $project->getID(),
-                      $dict['ctxt'],
-                      $dict['name'],
-                      $dict['type'],
-                      $dict['lang'],
-                      $dict['line'],
-                      $path_map[$dict['path']]);
-  }
-
-  if (!$no_purge) {
-    echo "Purging old syms...\n";
-    queryfx($conn_w,
-            'DELETE FROM %T WHERE arcanistProjectID = %d',
-            $symbol->getTableName(),
-            $project->getID());
-  }
-
-  echo "Loading ".number_format(count($sql))." syms...\n";
-  foreach (array_chunk($sql, 128) as $chunk) {
-    queryfx($conn_w,
-            'INSERT INTO %T
-      (arcanistProjectID, symbolContext, symbolName, symbolType,
-        symbolLanguage, lineNumber, pathID) VALUES %Q',
-            $symbol->getTableName(),
-            implode(', ', $chunk));
-  }
-
-}
-
-function check_string_value($value, $field_name, $line_no, $max_length) {
-   if (strlen($value) > $max_length) {
-      throw new Exception(
-        "{$field_name} '{$value}' defined on line #{$line_no} is too long, ".
-        "maximum {$field_name} length is {$max_length} characters.");
-    }
-
-    if (!phutil_is_utf8_with_only_bmp_characters($value)) {
-      throw new Exception(
-        "{$field_name} '{$value}' defined on line #{$line_no} is not a valid ".
-        "UTF-8 string, ".
-        "it should contain only UTF-8 characters.");
-    }
-}
-
-$no_purge = $args->getArg('no-purge');
-$symbols = array();
-foreach ($input as $key => $line) {
-  try {
-    $line_no = $key + 1;
-    $matches = null;
-    $ok = preg_match(
-      '/^((?P<context>[^ ]+)? )?(?P<name>[^ ]+) (?P<type>[^ ]+) '.
-      '(?P<lang>[^ ]+) (?P<line>\d+) (?P<path>.*)$/',
-      $line,
-      $matches);
-    if (!$ok) {
-      throw new Exception(
-        "Line #{$line_no} of input is invalid. Expected five or six ".
-        "space-delimited fields: maybe symbol context, symbol name, symbol ".
-        "type, symbol language, line number, path. ".
-        "For example:\n\n".
-        "idx function php 13 /path/to/some/file.php\n\n".
-        "Actual line was:\n\n".
-        "{$line}");
-    }
-    if (empty($matches['context'])) {
-      $matches['context'] = '';
-    }
-    $context     = $matches['context'];
-    $name        = $matches['name'];
-    $type        = $matches['type'];
-    $lang        = $matches['lang'];
-    $line_number = $matches['line'];
-    $path        = $matches['path'];
-
-    check_string_value($context, 'Symbol context', $line_no, 128);
-    check_string_value($name, 'Symbol name', $line_no, 128);
-    check_string_value($type, 'Symbol type', $line_no, 12);
-    check_string_value($lang, 'Symbol language', $line_no, 32);
-    check_string_value($path, 'Path', $line_no, 512);
-
-    if (!strlen($path) || $path[0] != '/') {
-      throw new Exception(
-        "Path '{$path}' defined on line #{$line_no} is invalid. Paths should ".
-        "begin with '/' and specify a path from the root of the project, like ".
-        "'/src/utils/utils.php'.");
-    }
-
-    $symbols[] = array(
-      'ctxt' => $context,
-      'name' => $name,
-      'type' => $type,
-      'lang' => $lang,
-      'line' => $line_number,
-      'path' => $path,
-    );
-  } catch (Exception $e) {
-    if ($args->getArg('ignore-errors')) {
-      continue;
-    } else {
-      throw $e;
-    }
-  }
-
-  if (count ($symbols) >= $args->getArg('max-transaction')) {
-      try {
-        echo "Committing {$args->getArg('max-transaction')} symbols....\n";
-        commit_symbols($symbols, $project, $no_purge);
-        $no_purge = true;
-        unset($symbols);
-        $symbols = array();
-      } catch (Exception $e) {
-        if ($args->getArg('ignore-errors')) {
-          continue;
-        } else {
-          throw $e;
-        }
-      }
-  }
-}
-
-if (count($symbols)) {
-  commit_symbols($symbols, $project, $no_purge);
-}
-
-echo "Done.\n";

scripts/symbols/import_repository_symbols.php

@@ -0,0 +1,228 @@
+#!/usr/bin/env php
+<?php
+
+$root = dirname(dirname(dirname(__FILE__)));
+require_once $root.'/scripts/__init_script__.php';
+
+$args = new PhutilArgumentParser($argv);
+$args->setSynopsis(<<<EOSYNOPSIS
+**import_repository_symbols.php** [__options__] __callsign__ < symbols
+
+  Import repository symbols (symbols are read from stdin).
+EOSYNOPSIS
+  );
+$args->parseStandardArguments();
+$args->parse(
+  array(
+    array(
+      'name'      => 'no-purge',
+      'help'      => pht(
+        'Do not clear all symbols for this repository before '.
+        'uploading new symbols. Useful for incremental updating.'),
+    ),
+    array(
+      'name'      => 'ignore-errors',
+      'help'      => pht(
+        "If a line can't be parsed, ignore that line and ".
+        "continue instead of exiting."),
+    ),
+    array(
+      'name'      => 'max-transaction',
+      'param'     => 'num-syms',
+      'default'   => '100000',
+      'help'      => pht(
+        'Maximum number of symbols that should '.
+        'be part of a single transaction.'),
+    ),
+    array(
+      'name'      => 'more',
+      'wildcard'  => true,
+    ),
+  ));
+
+$more = $args->getArg('more');
+if (count($more) !== 1) {
+  $args->printHelpAndExit();
+}
+
+$callsign   = head($more);
+$repository = id(new PhabricatorRepository())->loadOneWhere(
+  'callsign = %s',
+  $callsign);
+
+if (!$repository) {
+  echo pht("Repository '%s' does not exist.", $callsign);
+  exit(1);
+}
+
+if (!function_exists('posix_isatty') || posix_isatty(STDIN)) {
+  echo pht('Parsing input from stdin...'), "\n";
+}
+
+$input = file_get_contents('php://stdin');
+$input = trim($input);
+$input = explode("\n", $input);
+
+
+function commit_symbols(
+  array $symbols,
+  PhabricatorRepository $repository,
+  $no_purge) {
+
+  echo pht('Looking up path IDs...'), "\n";
+  $path_map =
+    PhabricatorRepositoryCommitChangeParserWorker::lookupOrCreatePaths(
+      ipull($symbols, 'path'));
+
+  $symbol = new PhabricatorRepositorySymbol();
+  $conn_w = $symbol->establishConnection('w');
+
+  echo pht('Preparing queries...'), "\n";
+  $sql = array();
+  foreach ($symbols as $dict) {
+    $sql[] = qsprintf(
+      $conn_w,
+      '(%s, %s, %s, %s, %s, %d, %d)',
+      $repository->getPHID(),
+      $dict['ctxt'],
+      $dict['name'],
+      $dict['type'],
+      $dict['lang'],
+      $dict['line'],
+      $path_map[$dict['path']]);
+  }
+
+  if (!$no_purge) {
+    echo pht('Purging old symbols...'), "\n";
+    queryfx(
+      $conn_w,
+      'DELETE FROM %T WHERE repositoryPHID = %s',
+      $symbol->getTableName(),
+      $repository->getPHID());
+  }
+
+  echo pht('Loading %s symbols...',  new PhutilNumber(count($sql))), "\n";
+  foreach (array_chunk($sql, 128) as $chunk) {
+    queryfx(
+      $conn_w,
+      'INSERT INTO %T
+        (repositoryPHID, symbolContext, symbolName, symbolType,
+        symbolLanguage, lineNumber, pathID) VALUES %Q',
+      $symbol->getTableName(),
+      implode(', ', $chunk));
+  }
+}
+
+function check_string_value($value, $field_name, $line_no, $max_length) {
+   if (strlen($value) > $max_length) {
+      throw new Exception(
+        pht(
+          "%s '%s' defined on line #%d is too long, ".
+          "maximum %s length is %d characters.",
+          $field_name,
+          $value,
+          $line_no,
+          $field_name,
+          $max_length));
+    }
+
+    if (!phutil_is_utf8_with_only_bmp_characters($value)) {
+      throw new Exception(
+        pht(
+          "%s '%s' defined on line #%d is not a valid ".
+          "UTF-8 string, it should contain only UTF-8 characters.",
+          $field_name,
+          $value,
+          $line_no));
+    }
+}
+
+$no_purge = $args->getArg('no-purge');
+$symbols = array();
+foreach ($input as $key => $line) {
+  try {
+    $line_no = $key + 1;
+    $matches = null;
+    $ok = preg_match(
+      '/^((?P<context>[^ ]+)? )?(?P<name>[^ ]+) (?P<type>[^ ]+) '.
+      '(?P<lang>[^ ]+) (?P<line>\d+) (?P<path>.*)$/',
+      $line,
+      $matches);
+    if (!$ok) {
+      throw new Exception(
+        pht(
+          "Line #%d of input is invalid. Expected five or six space-delimited ".
+          "fields: maybe symbol context, symbol name, symbol type, symbol ".
+          "language, line number, path. For example:\n\n%s\n\n".
+          "Actual line was:\n\n%s",
+          $line_no,
+          'idx function php 13 /path/to/some/file.php',
+          $line));
+    }
+    if (empty($matches['context'])) {
+      $matches['context'] = '';
+    }
+    $context     = $matches['context'];
+    $name        = $matches['name'];
+    $type        = $matches['type'];
+    $lang        = $matches['lang'];
+    $line_number = $matches['line'];
+    $path        = $matches['path'];
+
+    check_string_value($context, 'Symbol context', $line_no, 128);
+    check_string_value($name, 'Symbol name', $line_no, 128);
+    check_string_value($type, 'Symbol type', $line_no, 12);
+    check_string_value($lang, 'Symbol language', $line_no, 32);
+    check_string_value($path, 'Path', $line_no, 512);
+
+    if (!strlen($path) || $path[0] != '/') {
+      throw new Exception(
+        pht(
+          "Path '%s' defined on line #%d is invalid. Paths should begin with ".
+          "'%s' and specify a path from the root of the project, like '%s'.",
+          $path,
+          $line_no,
+          '/',
+          '/src/utils/utils.php'));
+    }
+
+    $symbols[] = array(
+      'ctxt' => $context,
+      'name' => $name,
+      'type' => $type,
+      'lang' => $lang,
+      'line' => $line_number,
+      'path' => $path,
+    );
+  } catch (Exception $e) {
+    if ($args->getArg('ignore-errors')) {
+      continue;
+    } else {
+      throw $e;
+    }
+  }
+
+  if (count ($symbols) >= $args->getArg('max-transaction')) {
+    try {
+      echo pht(
+        "Committing %s symbols...\n",
+        new PhutilNumber($args->getArg('max-transaction')));
+      commit_symbols($symbols, $repository, $no_purge);
+      $no_purge = true;
+      unset($symbols);
+      $symbols = array();
+    } catch (Exception $e) {
+      if ($args->getArg('ignore-errors')) {
+        continue;
+      } else {
+        throw $e;
+      }
+    }
+  }
+}
+
+if (count($symbols)) {
+  commit_symbols($symbols, $repository, $no_purge);
+}
+
+echo pht('Done.'), "\n";

src/applications/diffusion/conduit/DiffusionFindSymbolsConduitAPIMethod.php

@@ -8,7 +8,7 @@ final class DiffusionFindSymbolsConduitAPIMethod
   }
 
   public function getMethodDescription() {
-    return 'Retrieve Diffusion symbol information.';
+    return pht('Retrieve Diffusion symbol information.');
   }
 
   protected function defineParamTypes() {
@@ -51,7 +51,6 @@ final class DiffusionFindSymbolsConduitAPIMethod
     }
 
     $query->needPaths(true);
-    $query->needArcanistProjects(true);
     $query->needRepositories(true);
 
     $results = $query->execute();

src/applications/diffusion/controller/DiffusionSymbolController.php

@@ -24,25 +24,25 @@ final class DiffusionSymbolController extends DiffusionController {
       $query->setLanguage($request->getStr('lang'));
     }
 
-    if ($request->getStr('projects')) {
+    if ($request->getStr('repositories')) {
-      $phids = $request->getStr('projects');
+      $phids = $request->getStr('repositories');
       $phids = explode(',', $phids);
       $phids = array_filter($phids);
 
       if ($phids) {
-        $projects = id(new PhabricatorRepositoryArcanistProject())
+        $repos = id(new PhabricatorRepositoryQuery())
-          ->loadAllWhere(
+          ->setViewer($request->getUser())
-            'phid IN (%Ls)',
+          ->withPHIDs($phids)
-            $phids);
+          ->execute();
-        $projects = mpull($projects, 'getID');
+
-        if ($projects) {
+        $repos = mpull($repos, 'getPHID');
-          $query->setProjectIDs($projects);
+        if ($repos) {
+          $query->withRepositoryPHIDs($repos);
         }
       }
     }
 
     $query->needPaths(true);
-    $query->needArcanistProjects(true);
     $query->needRepositories(true);
 
     $symbols = $query->execute();
@@ -73,13 +73,6 @@ final class DiffusionSymbolController extends DiffusionController {
 
     $rows = array();
     foreach ($symbols as $symbol) {
-      $project = $symbol->getArcanistProject();
-      if ($project) {
-        $project_name = $project->getName();
-      } else {
-        $project_name = '-';
-      }
-
       $file = $symbol->getPath();
       $line = $symbol->getLineNumber();
 
@@ -110,7 +103,7 @@ final class DiffusionSymbolController extends DiffusionController {
         $symbol->getSymbolContext(),
         $symbol->getSymbolName(),
         $symbol->getSymbolLanguage(),
-        $project_name,
+        $repo->getMonogram(),
         $location,
       );
     }
@@ -122,7 +115,7 @@ final class DiffusionSymbolController extends DiffusionController {
         pht('Context'),
         pht('Name'),
         pht('Language'),
-        pht('Project'),
+        pht('Repository'),
         pht('File'),
       ));
     $table->setColumnClasses(

src/applications/diffusion/query/DiffusionSymbolQuery.php

@@ -16,12 +16,11 @@ final class DiffusionSymbolQuery extends PhabricatorOffsetPagedQuery {
   private $namePrefix;
   private $name;
 
-  private $projectIDs;
+  private $repositoryPHIDs;
   private $language;
   private $type;
 
   private $needPaths;
-  private $needArcanistProject;
   private $needRepositories;
 
 
@@ -72,8 +71,8 @@ final class DiffusionSymbolQuery extends PhabricatorOffsetPagedQuery {
   /**
    * @task config
    */
-  public function setProjectIDs(array $project_ids) {
+  public function withRepositoryPHIDs(array $repository_phids) {
-    $this->projectIDs = $project_ids;
+    $this->repositoryPHIDs = $repository_phids;
     return $this;
   }
 
@@ -105,15 +104,6 @@ final class DiffusionSymbolQuery extends PhabricatorOffsetPagedQuery {
   }
 
 
-  /**
-   * @task config
-   */
-  public function needArcanistProjects($need_arcanist_projects) {
-    $this->needArcanistProjects = $need_arcanist_projects;
-    return $this;
-  }
-
-
   /**
    * @task config
    */
@@ -132,10 +122,10 @@ final class DiffusionSymbolQuery extends PhabricatorOffsetPagedQuery {
   public function execute() {
     if ($this->name && $this->namePrefix) {
       throw new Exception(
-        'You can not set both a name and a name prefix!');
+        pht('You can not set both a name and a name prefix!'));
     } else if (!$this->name && !$this->namePrefix) {
       throw new Exception(
-        'You must set a name or a name prefix!');
+        pht('You must set a name or a name prefix!'));
     }
 
     $symbol = new PhabricatorRepositorySymbol();
@@ -155,9 +145,6 @@ final class DiffusionSymbolQuery extends PhabricatorOffsetPagedQuery {
       if ($this->needPaths) {
         $this->loadPaths($symbols);
       }
-      if ($this->needArcanistProjects || $this->needRepositories) {
-        $this->loadArcanistProjects($symbols);
-      }
       if ($this->needRepositories) {
         $this->loadRepositories($symbols);
       }
@@ -208,11 +195,11 @@ final class DiffusionSymbolQuery extends PhabricatorOffsetPagedQuery {
         $this->namePrefix);
     }
 
-    if ($this->projectIDs) {
+    if ($this->repositoryPHIDs) {
       $where[] = qsprintf(
         $conn_r,
-        'arcanistProjectID IN (%Ld)',
+        'repositoryPHID IN (%Ls)',
-        $this->projectIDs);
+        $this->repositoryPHIDs);
     }
 
     if ($this->language) {
@@ -250,49 +237,21 @@ final class DiffusionSymbolQuery extends PhabricatorOffsetPagedQuery {
   }
 
 
-  /**
-   * @task internal
-   */
-  private function loadArcanistProjects(array $symbols) {
-    assert_instances_of($symbols, 'PhabricatorRepositorySymbol');
-    $projects = id(new PhabricatorRepositoryArcanistProject())->loadAllWhere(
-      'id IN (%Ld)',
-      mpull($symbols, 'getArcanistProjectID'));
-    foreach ($symbols as $symbol) {
-      $project = idx($projects, $symbol->getArcanistProjectID());
-      $symbol->attachArcanistProject($project);
-    }
-  }
-
-
   /**
    * @task internal
    */
   private function loadRepositories(array $symbols) {
     assert_instances_of($symbols, 'PhabricatorRepositorySymbol');
 
-    $projects = mpull($symbols, 'getArcanistProject');
-    $projects = array_filter($projects);
-
-    $repo_ids = mpull($projects, 'getRepositoryID');
-    $repo_ids = array_filter($repo_ids);
-
-    if ($repo_ids) {
     $repos = id(new PhabricatorRepositoryQuery())
-        ->setViewer($this->getViewer())
+      ->setViewer($this->viewer)
-        ->withIDs($repo_ids)
+      ->withPHIDs(mpull($symbols, 'getRepositoryPHID'))
       ->execute();
-    } else {
+    $repos = mpull($repos, null, 'getPHID');
-      $repos = array();
-    }
 
     foreach ($symbols as $symbol) {
-      $proj = $symbol->getArcanistProject();
+      $repository = idx($repos, $symbol->getRepositoryPHID());
-      if ($proj) {
+      $symbol->attachRepository($repository);
-        $symbol->attachRepository(idx($repos, $proj->getRepositoryID()));
-      } else {
-        $symbol->attachRepository(null);
-      }
     }
   }
 

src/applications/diffusion/typeahead/DiffusionSymbolDatasource.php

@@ -32,7 +32,6 @@ final class DiffusionSymbolDatasource
         ->setViewer($viewer)
         ->setNamePrefix($raw_query)
         ->setLimit(15)
-        ->needArcanistProjects(true)
         ->needRepositories(true)
         ->needPaths(true)
         ->execute();
@@ -40,14 +39,14 @@ final class DiffusionSymbolDatasource
         $lang = $symbol->getSymbolLanguage();
         $name = $symbol->getSymbolName();
         $type = $symbol->getSymbolType();
-        $proj = $symbol->getArcanistProject()->getName();
+        $repo = $symbol->getRepository()->getName();
 
         $results[] = id(new PhabricatorTypeaheadResult())
           ->setName($name)
           ->setURI($symbol->getURI())
           ->setPHID(md5($symbol->getURI())) // Just needs to be unique.
           ->setDisplayName($name)
-          ->setDisplayType(strtoupper($lang).' '.ucwords($type).' ('.$proj.')')
+          ->setDisplayType(strtoupper($lang).' '.ucwords($type).' ('.$repo.')')
           ->setPriorityType('symb');
       }
     }

src/applications/repository/storage/PhabricatorRepository.php

@@ -1163,10 +1163,15 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
       $projects = id(new PhabricatorRepositoryArcanistProject())
         ->loadAllWhere('repositoryID = %d', $this->getID());
       foreach ($projects as $project) {
-        // note each project deletes its PhabricatorRepositorySymbols
         $project->delete();
       }
 
+      queryfx(
+        $this->establishConnection('w'),
+        'DELETE FROM %T WHERE repositoryPHID = %s',
+        id(new PhabricatorRepositorySymbol())->getTableName(),
+        $this->getPHID());
+
       $commits = id(new PhabricatorRepositoryCommit())
         ->loadAllWhere('repositoryID = %d', $this->getID());
       foreach ($commits as $commit) {

src/applications/repository/storage/PhabricatorRepositoryArcanistProject.php

@@ -45,20 +45,6 @@ final class PhabricatorRepositoryArcanistProject
       PhabricatorRepositoryArcanistProjectPHIDType::TYPECONST);
   }
 
-  public function delete() {
-    $this->openTransaction();
-
-      queryfx(
-        $this->establishConnection('w'),
-        'DELETE FROM %T WHERE arcanistProjectID = %d',
-        id(new PhabricatorRepositorySymbol())->getTableName(),
-        $this->getID());
-
-      $result = parent::delete();
-    $this->saveTransaction();
-    return $result;
-  }
-
   public function getRepository() {
     return $this->assertAttached($this->repository);
   }

src/applications/repository/storage/PhabricatorRepositorySymbol.php

@@ -8,7 +8,7 @@
  */
 final class PhabricatorRepositorySymbol extends PhabricatorRepositoryDAO {
 
-  protected $arcanistProjectID;
+  protected $repositoryPHID;
   protected $symbolContext;
   protected $symbolName;
   protected $symbolType;
@@ -17,12 +17,10 @@ final class PhabricatorRepositorySymbol extends PhabricatorRepositoryDAO {
   protected $lineNumber;
 
   private $path = self::ATTACHABLE;
-  private $arcanistProject = self::ATTACHABLE;
   private $repository = self::ATTACHABLE;
 
   protected function getConfiguration() {
     return array(
-      self::CONFIG_IDS => self::IDS_MANUAL,
       self::CONFIG_TIMESTAMPS => false,
       self::CONFIG_COLUMN_SCHEMA => array(
         'id' => null,
@@ -42,13 +40,6 @@ final class PhabricatorRepositorySymbol extends PhabricatorRepositoryDAO {
   }
 
   public function getURI() {
-    if (!$this->repository) {
-      // This symbol is in the index, but we don't know which Repository it's
-      // part of. Usually this means the Arcanist Project hasn't been linked
-      // to a Repository. We can't generate a URI, so just fail.
-      return null;
-    }
-
     $request = DiffusionRequest::newFromDictionary(
       array(
         'user' => PhabricatorUser::getOmnipotentUser(),
@@ -75,18 +66,9 @@ final class PhabricatorRepositorySymbol extends PhabricatorRepositoryDAO {
     return $this->assertAttached($this->repository);
   }
 
-  public function attachRepository($repository) {
+  public function attachRepository(PhabricatorRepository $repository) {
     $this->repository = $repository;
     return $this;
   }
 
-  public function getArcanistProject() {
-    return $this->assertAttached($this->arcanistProject);
-  }
-
-  public function attachArcanistProject($project) {
-    $this->arcanistProject = $project;
-    return $this;
-  }
-
 }