Framework for external symbol search

Summary:
Ref T7984. With this, an install can add an ExternalSymbolsSource to src/extensions, which will include whatever
source they have.

Test Plan: search for php and python builtins.

Reviewers: joshuaspence, epriestley, #blessed_reviewers

Reviewed By: epriestley, #blessed_reviewers

Subscribers: Korvin, epriestley

Maniphest Tasks: T7984

Differential Revision: https://secure.phabricator.com/D13036

src/__phutil_library_map__.php

@@ -507,6 +507,8 @@ phutil_register_library_map(array(
     'DiffusionEmptyResultView' => 'applications/diffusion/view/DiffusionEmptyResultView.php',
     'DiffusionExistsQueryConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionExistsQueryConduitAPIMethod.php',
     'DiffusionExternalController' => 'applications/diffusion/controller/DiffusionExternalController.php',
+    'DiffusionExternalSymbolQuery' => 'applications/diffusion/symbol/DiffusionExternalSymbolQuery.php',
+    'DiffusionExternalSymbolsSource' => 'applications/diffusion/symbol/DiffusionExternalSymbolsSource.php',
     'DiffusionFileContent' => 'applications/diffusion/data/DiffusionFileContent.php',
     'DiffusionFileContentQuery' => 'applications/diffusion/query/filecontent/DiffusionFileContentQuery.php',
     'DiffusionFileContentQueryConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionFileContentQueryConduitAPIMethod.php',
@@ -565,11 +567,13 @@ phutil_register_library_map(array(
     'DiffusionPathQueryTestCase' => 'applications/diffusion/query/pathid/__tests__/DiffusionPathQueryTestCase.php',
     'DiffusionPathTreeController' => 'applications/diffusion/controller/DiffusionPathTreeController.php',
     'DiffusionPathValidateController' => 'applications/diffusion/controller/DiffusionPathValidateController.php',
+    'DiffusionPhpExternalSymbolsSource' => 'applications/diffusion/symbol/DiffusionPhpExternalSymbolsSource.php',
     'DiffusionPushCapability' => 'applications/diffusion/capability/DiffusionPushCapability.php',
     'DiffusionPushEventViewController' => 'applications/diffusion/controller/DiffusionPushEventViewController.php',
     'DiffusionPushLogController' => 'applications/diffusion/controller/DiffusionPushLogController.php',
     'DiffusionPushLogListController' => 'applications/diffusion/controller/DiffusionPushLogListController.php',
     'DiffusionPushLogListView' => 'applications/diffusion/view/DiffusionPushLogListView.php',
+    'DiffusionPythonExternalSymbolsSource' => 'applications/diffusion/symbol/DiffusionPythonExternalSymbolsSource.php',
     'DiffusionQuery' => 'applications/diffusion/query/DiffusionQuery.php',
     'DiffusionQueryCommitsConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionQueryCommitsConduitAPIMethod.php',
     'DiffusionQueryConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionQueryConduitAPIMethod.php',
@@ -3803,11 +3807,13 @@ phutil_register_library_map(array(
     'DiffusionPathQueryTestCase' => 'PhabricatorTestCase',
     'DiffusionPathTreeController' => 'DiffusionController',
     'DiffusionPathValidateController' => 'DiffusionController',
+    'DiffusionPhpExternalSymbolsSource' => 'DiffusionExternalSymbolsSource',
     'DiffusionPushCapability' => 'PhabricatorPolicyCapability',
     'DiffusionPushEventViewController' => 'DiffusionPushLogController',
     'DiffusionPushLogController' => 'DiffusionController',
     'DiffusionPushLogListController' => 'DiffusionPushLogController',
     'DiffusionPushLogListView' => 'AphrontView',
+    'DiffusionPythonExternalSymbolsSource' => 'DiffusionExternalSymbolsSource',
     'DiffusionQuery' => 'PhabricatorQuery',
     'DiffusionQueryCommitsConduitAPIMethod' => 'DiffusionConduitAPIMethod',
     'DiffusionQueryConduitAPIMethod' => 'DiffusionConduitAPIMethod',

src/applications/diffusion/controller/DiffusionSymbolController.php

@@ -12,7 +12,7 @@ final class DiffusionSymbolController extends DiffusionController {
       ->setViewer($user)
       ->setName($this->name);
 
-    if ($request->getStr('context') !== null) {
+    if ($request->getStr('context')) {
       $query->setContext($request->getStr('context'));
     }
 
@@ -47,63 +47,69 @@ final class DiffusionSymbolController extends DiffusionController {
 
     $symbols = $query->execute();
 
-    // For PHP builtins, jump to php.net documentation.
-    if ($request->getBool('jump') && count($symbols) == 0) {
-      if ($request->getStr('lang', 'php') == 'php') {
-        if ($request->getStr('type', 'function') == 'function') {
-          $functions = get_defined_functions();
-          if (in_array($this->name, $functions['internal'])) {
-            return id(new AphrontRedirectResponse())
-              ->setIsExternal(true)
-              ->setURI('http://www.php.net/function.'.$this->name);
-          }
-        }
-        if ($request->getStr('type', 'class') == 'class') {
-          if (class_exists($this->name, false) ||
-              interface_exists($this->name, false)) {
-            if (id(new ReflectionClass($this->name))->isInternal()) {
-              return id(new AphrontRedirectResponse())
-                ->setIsExternal(true)
-                ->setURI('http://www.php.net/class.'.$this->name);
-            }
-          }
-        }
-      }
+
+
+    $external_query = id(new DiffusionExternalSymbolQuery())
+      ->withNames(array($this->name));
+
+    if ($request->getStr('context')) {
+      $external_query->withContexts(array($request->getStr('context')));
+    }
+
+    if ($request->getStr('type')) {
+      $external_query->withTypes(array($request->getStr('type')));
+    }
+
+    if ($request->getStr('lang')) {
+      $external_query->withLanguages(array($request->getStr('lang')));
+    }
+
+    $external_sources = id(new PhutilSymbolLoader())
+      ->setAncestorClass('DiffusionExternalSymbolsSource')
+      ->loadObjects();
+    $results = array($symbols);
+    foreach ($external_sources as $source) {
+      $results[] = $source->executeQuery($external_query);
+    }
+    $symbols = array_mergev($results);
+
+    if ($request->getBool('jump') && count($symbols) == 1) {
+      // If this is a clickthrough from Differential, just jump them
+      // straight to the target if we got a single hit.
+      $symbol = head($symbols);
+      return id(new AphrontRedirectResponse())
+        ->setIsExternal($symbol->isExternal())
+        ->setURI($symbol->getURI());
     }
 
     $rows = array();
     foreach ($symbols as $symbol) {
-      $file = $symbol->getPath();
-      $line = $symbol->getLineNumber();
+      $href = $symbol->getURI();
 
-      $repo = $symbol->getRepository();
-      if ($repo) {
-        $href = $symbol->getURI();
-
-        if ($request->getBool('jump') && count($symbols) == 1) {
-          // If this is a clickthrough from Differential, just jump them
-          // straight to the target if we got a single hit.
-          return id(new AphrontRedirectResponse())->setURI($href);
-        }
-
-        $location = phutil_tag(
-          'a',
-          array(
-            'href' => $href,
-          ),
-          $file.':'.$line);
-      } else if ($file) {
-        $location = $file.':'.$line;
+      if ($symbol->isExternal()) {
+        $source = $symbol->getSource();
+        $location = $symbol->getLocation();
       } else {
-        $location = '?';
+        $repo = $symbol->getRepository();
+        $file = $symbol->getPath();
+        $line = $symbol->getLineNumber();
+
+        $source = $repo->getMonogram();
+        $location = $file.':'.$line;
       }
+      $location = phutil_tag(
+        'a',
+        array(
+          'href' => $href,
+        ),
+        $location);
 
       $rows[] = array(
         $symbol->getSymbolType(),
         $symbol->getSymbolContext(),
         $symbol->getSymbolName(),
         $symbol->getSymbolLanguage(),
-        $repo->getMonogram(),
+        $source,
         $location,
       );
     }
@@ -115,8 +121,8 @@ final class DiffusionSymbolController extends DiffusionController {
         pht('Context'),
         pht('Name'),
         pht('Language'),
-        pht('Repository'),
-        pht('File'),
+        pht('Source'),
+        pht('Location'),
       ));
     $table->setColumnClasses(
       array(

src/applications/diffusion/symbol/DiffusionExternalSymbolQuery.php

@@ -0,0 +1,46 @@
+<?php
+
+final class DiffusionExternalSymbolQuery {
+  private $languages = array();
+  private $types = array();
+  private $names = array();
+  private $contexts = array();
+
+  public function withLanguages(array $languages) {
+    $this->languages = $languages;
+    return $this;
+  }
+  public function withTypes(array $types) {
+    $this->types = $types;
+    return $this;
+  }
+  public function withNames(array $names) {
+    $this->names = $names;
+    return $this;
+  }
+  public function withContexts(array $contexts) {
+    $this->contexts = $contexts;
+    return $this;
+  }
+
+
+  public function getLanguages() {
+    return $this->languages;
+  }
+  public function getTypes() {
+    return $this->types;
+  }
+  public function getNames() {
+    return $this->names;
+  }
+  public function getContexts() {
+    return $this->contexts;
+  }
+
+  public function matchesAnyLanguage(array $languages) {
+    return (!$this->languages) || array_intersect($languages, $this->languages);
+  }
+  public function matchesAnyType(array $types) {
+    return (!$this->types) || array_intersect($types, $this->types);
+  }
+}

src/applications/diffusion/symbol/DiffusionExternalSymbolsSource.php

@@ -0,0 +1,15 @@
+<?php
+
+abstract class DiffusionExternalSymbolsSource {
+
+  /**
+   * @return list of PhabricatorRepositorySymbol
+   */
+  abstract public function executeQuery(DiffusionExternalSymbolQuery $query);
+
+  protected function buildExternalSymbol() {
+    return id(new PhabricatorRepositorySymbol())
+      ->setIsExternal(true)
+      ->makeEphemeral();
+  }
+}

src/applications/diffusion/symbol/DiffusionPhpExternalSymbolsSource.php

@@ -0,0 +1,49 @@
+<?php
+
+final class DiffusionPhpExternalSymbolsSource
+  extends DiffusionExternalSymbolsSource {
+
+  public function executeQuery(DiffusionExternalSymbolQuery $query) {
+    $symbols = array();
+
+    if (!$query->matchesAnyLanguage(array('php'))) {
+      return $symbols;
+    }
+
+    $names = $query->getNames();
+
+    if ($query->matchesAnyType(array('function'))) {
+      $functions = get_defined_functions();
+      $functions = $functions['internal'];
+
+      foreach ($names as $name) {
+        if (in_array($name, $functions)) {
+          $symbols[] = $this->buildExternalSymbol()
+            ->setSymbolName($name)
+            ->setSymbolType('function')
+            ->setSource(pht('PHP'))
+            ->setLocation(pht('Manual at php.net'))
+            ->setSymbolLanguage('php')
+            ->setExternalURI('http://www.php.net/function.'.$name);
+        }
+      }
+    }
+    if ($query->matchesAnyType(array('class'))) {
+      foreach ($names as $name) {
+        if (class_exists($name, false) || interface_exists($name, false)) {
+          if (id(new ReflectionClass($name))->isInternal()) {
+            $symbols[] = $this->buildExternalSymbol()
+              ->setSymbolName($name)
+              ->setSymbolType('class')
+              ->setSource(pht('PHP'))
+              ->setLocation(pht('Manual at php.net'))
+              ->setSymbolLanguage('php')
+              ->setExternalURI('http://www.php.net/class.'.$name);
+          }
+        }
+      }
+    }
+
+    return $symbols;
+  }
+}

src/applications/diffusion/symbol/DiffusionPythonExternalSymbolsSource.php

@@ -0,0 +1,134 @@
+<?php
+
+final class DiffusionPythonExternalSymbolsSource
+  extends DiffusionExternalSymbolsSource {
+
+  public function executeQuery(DiffusionExternalSymbolQuery $query) {
+    $symbols = array();
+    if (!$query->matchesAnyLanguage(array('py', 'python'))) {
+      return $symbols;
+    }
+
+    if (!$query->matchesAnyType(array('builtin', 'function'))) {
+      return $symbols;
+    }
+
+    $names = $query->getNames();
+
+    foreach ($names as $name) {
+      if (idx(self::$python2Builtins, $name)) {
+        $symbols[] = $this->buildExternalSymbol()
+          ->setSymbolName($name)
+          ->setSymbolType('function')
+          ->setSource(pht('Standard Library'))
+          ->setLocation(pht('The Python 2 Standard Library'))
+          ->setSymbolLanguage('py')
+          ->setExternalURI(
+            'https://docs.python.org/2/library/functions.html#'.$name);
+      }
+      if (idx(self::$python3Builtins, $name)) {
+        $symbols[] = $this->buildExternalSymbol()
+          ->setSymbolName($name)
+          ->setSymbolType('function')
+          ->setSource(pht('Standard Library'))
+          ->setLocation(pht('The Python 3 Standard Library'))
+          ->setSymbolLanguage('py')
+          ->setExternalURI(
+            'https://docs.python.org/3/library/functions.html#'.$name);
+      }
+    }
+    return $symbols;
+  }
+
+  private static $python2Builtins = array(
+    '__import__' => true,
+    'abs' => true,
+    'all' => true,
+    'any' => true,
+    'basestring' => true,
+    'bin' => true,
+    'bool' => true,
+    'bytearray' => true,
+    'callable' => true,
+    'chr' => true,
+    'classmethod' => true,
+    'cmp' => true,
+    'compile' => true,
+    'complex' => true,
+    'delattr' => true,
+    'dict' => true,
+    'dir' => true,
+    'divmod' => true,
+    'enumerate' => true,
+    'eval' => true,
+    'execfile' => true,
+    'file' => true,
+    'filter' => true,
+    'float' => true,
+    'format' => true,
+    'frozenset' => true,
+    'getattr' => true,
+    'globals' => true,
+    'hasattr' => true,
+    'hash' => true,
+    'help' => true,
+    'hex' => true,
+    'id' => true,
+    'input' => true,
+    'int' => true,
+    'isinstance' => true,
+    'issubclass' => true,
+    'iter' => true,
+    'len' => true,
+    'list' => true,
+    'locals' => true,
+    'long' => true,
+    'map' => true,
+    'max' => true,
+    'memoryview' => true,
+    'min' => true,
+    'next' => true,
+    'object' => true,
+    'oct' => true,
+    'open' => true,
+    'ord' => true,
+    'pow' => true,
+    'print' => true,
+    'property' => true,
+    'range' => true,
+    'raw_input' => true,
+    'reduce' => true,
+    'reload' => true,
+    'repr' => true,
+    'reversed' => true,
+    'round' => true,
+    'set' => true,
+    'setattr' => true,
+    'slice' => true,
+    'sorted' => true,
+    'staticmethod' => true,
+    'str' => true,
+    'sum' => true,
+    'super' => true,
+    'tuple' => true,
+    'type' => true,
+    'unichr' => true,
+    'unicode' => true,
+    'vars' => true,
+    'xrange' => true,
+    'zip' => true,
+  );
+
+  // This list only contains functions that are new or changed between the
+  // Python versions.
+  private static $python3Builtins = array(
+    'ascii' => true,
+    'bytes' => true,
+    'filter' => true,
+    'map' => true,
+    'next' => true,
+    'range' => true,
+    'super' => true,
+    'zip' => true,
+  );
+}

src/applications/repository/storage/PhabricatorRepositorySymbol.php

@@ -15,6 +15,10 @@ final class PhabricatorRepositorySymbol extends PhabricatorRepositoryDAO {
   protected $symbolLanguage;
   protected $pathID;
   protected $lineNumber;
+  private $isExternal;
+  private $source;
+  private $location;
+  private $externalURI;
 
   private $path = self::ATTACHABLE;
   private $repository = self::ATTACHABLE;
@@ -40,6 +44,10 @@ final class PhabricatorRepositorySymbol extends PhabricatorRepositoryDAO {
   }
 
   public function getURI() {
+    if ($this->isExternal) {
+      return $this->externalURI;
+    }
+
     $request = DiffusionRequest::newFromDictionary(
       array(
         'user' => PhabricatorUser::getOmnipotentUser(),
@@ -71,4 +79,32 @@ final class PhabricatorRepositorySymbol extends PhabricatorRepositoryDAO {
     return $this;
   }
 
+  public function isExternal() {
+    return $this->isExternal;
+  }
+  public function setIsExternal($is_external) {
+    $this->isExternal = $is_external;
+    return $this;
+  }
+
+  public function getSource() {
+    return $this->source;
+  }
+  public function setSource($source) {
+    $this->source = $source;
+    return $this;
+  }
+
+  public function getLocation() {
+    return $this->location;
+  }
+  public function setLocation($location) {
+    $this->location = $location;
+    return $this;
+  }
+
+  public function setExternalURI($external_uri) {
+    $this->externalURI = $external_uri;
+    return $this;
+  }
 }