Index all repository URIs, not just the "primary" repository URI

Summary:
Ref T10923. When regenerating the URI index for a repository, index every URI.

  - Also, make the index slightly stricter (domain + path instead of just path). Excluding the domain made more sense when we were generating only first-party URIs.
  - Make the index smarter about `/diffusion/123/` URIs.
  - Show normalized URIs in `diffusion.repository.search` results.

Test Plan:
  - Ran migration.
  - Verified sensible-looking results in database.
  - Searched for a repository URI by first-party clone URI.
  - Searched for a repository URI by mirror URI.
  - Used `diffusion.repository.search` to get information about repository URIs.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T10923

Differential Revision: https://secure.phabricator.com/D15876

resources/sql/autopatches/20160112.repo.02.uri.index.php

@@ -1,7 +1,4 @@
 <?php
 
-$table = new PhabricatorRepository();
+// A later patch ("20160510.repo.01.uriindex.php") performs an identical
-
+// regeneration of the index, so we no longer need to do it here.
-foreach (new LiskMigrationIterator($table) as $repo) {
-  $repo->updateURIIndex();
-}

resources/sql/autopatches/20160510.repo.01.uriindex.php

@@ -0,0 +1,10 @@
+<?php
+
+$repos = id(new PhabricatorRepositoryQuery())
+  ->setViewer(PhabricatorUser::getOmnipotentUser())
+  ->needURIs(true)
+  ->execute();
+
+foreach ($repos as $repo) {
+  $repo->updateURIIndex();
+}

src/applications/repository/data/PhabricatorRepositoryURINormalizer.php

@@ -59,6 +59,14 @@ final class PhabricatorRepositoryURINormalizer extends Phobject {
     $this->uri = $uri;
   }
 
+  public static function getAllURITypes() {
+    return array(
+      self::TYPE_GIT,
+      self::TYPE_SVN,
+      self::TYPE_MERCURIAL,
+    );
+  }
+
 
 /* -(  Normalizing URIs  )--------------------------------------------------- */
 
@@ -91,6 +99,10 @@ final class PhabricatorRepositoryURINormalizer extends Phobject {
     }
   }
 
+  public function getNormalizedURI() {
+    return $this->getNormalizedDomain().'/'.$this->getNormalizedPath();
+  }
+
 
   /**
    * @task normal
@@ -113,11 +125,32 @@ final class PhabricatorRepositoryURINormalizer extends Phobject {
     // example.
 
     $matches = null;
-    if (preg_match('@^(diffusion/[A-Z]+)@', $path, $matches)) {
+    if (preg_match('@^(diffusion/(?:[A-Z]+|\d+))@', $path, $matches)) {
       $path = $matches[1];
     }
 
     return $path;
   }
 
+  public function getNormalizedDomain() {
+    $domain = null;
+
+    $uri = new PhutilURI($this->uri);
+    if ($uri->getProtocol()) {
+      $domain = $uri->getDomain();
+    }
+
+    if (!strlen($domain)) {
+      $uri = new PhutilGitURI($this->uri);
+      $domain = $uri->getDomain();
+    }
+
+    if (!strlen($domain)) {
+      $domain = '<void>';
+    }
+
+    return phutil_utf8_strtolower($domain);
+  }
+
+
 }

src/applications/repository/query/PhabricatorRepositoryQuery.php

@@ -650,7 +650,7 @@ final class PhabricatorRepositoryQuery
     }
 
     if ($this->uris !== null) {
-      $try_uris = $this->getNormalizedPaths();
+      $try_uris = $this->getNormalizedURIs();
       $try_uris = array_fuse($try_uris);
 
       $where[] = qsprintf(
@@ -666,7 +666,7 @@ final class PhabricatorRepositoryQuery
     return 'PhabricatorDiffusionApplication';
   }
 
-  private function getNormalizedPaths() {
+  private function getNormalizedURIs() {
     $normalized_uris = array();
 
     // Since we don't know which type of repository this URI is in the general
@@ -675,19 +675,15 @@ final class PhabricatorRepositoryQuery
     // or an `svn+ssh` URI, we could deduce how to normalize it. However, this
     // would be more complicated and it's not clear if it matters in practice.
 
+    $types = PhabricatorRepositoryURINormalizer::getAllURITypes();
     foreach ($this->uris as $uri) {
-      $normalized_uris[] = new PhabricatorRepositoryURINormalizer(
+      foreach ($types as $type) {
-        PhabricatorRepositoryURINormalizer::TYPE_GIT,
+        $normalized_uri = new PhabricatorRepositoryURINormalizer($type, $uri);
-        $uri);
+        $normalized_uris[] = $normalized_uri->getNormalizedURI();
-      $normalized_uris[] = new PhabricatorRepositoryURINormalizer(
+      }
-        PhabricatorRepositoryURINormalizer::TYPE_SVN,
-        $uri);
-      $normalized_uris[] = new PhabricatorRepositoryURINormalizer(
-        PhabricatorRepositoryURINormalizer::TYPE_MERCURIAL,
-        $uri);
     }
 
-    return array_unique(mpull($normalized_uris, 'getNormalizedPath'));
+    return array_unique($normalized_uris);
   }
 
 }

src/applications/repository/query/PhabricatorRepositorySearchEngine.php

@@ -38,6 +38,11 @@ final class PhabricatorRepositorySearchEngine
         ->setLabel(pht('Types'))
         ->setKey('types')
         ->setOptions(PhabricatorRepositoryType::getAllRepositoryTypes()),
+      id(new PhabricatorSearchStringListField())
+        ->setLabel(pht('URIs'))
+        ->setKey('uris')
+        ->setDescription(
+          pht('Search for repositories by clone/checkout URI.')),
     );
   }
 
@@ -70,6 +75,10 @@ final class PhabricatorRepositorySearchEngine
       $query->withNameContains($map['name']);
     }
 
+    if ($map['uris']) {
+      $query->withURIs($map['uris']);
+    }
+
     return $query;
   }
 

src/applications/repository/storage/PhabricatorRepository.php

@@ -803,41 +803,24 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
   }
 
   public function updateURIIndex() {
-    $uris = array(
+    $indexes = array();
-      (string)$this->getCloneURIObject(),
-    );
 
-    foreach ($uris as $key => $uri) {
+    $uris = $this->getURIs();
-      $uris[$key] = $this->getNormalizedURI($uri)
+    foreach ($uris as $uri) {
-        ->getNormalizedPath();
+      if ($uri->getIsDisabled()) {
+        continue;
+      }
+
+      $indexes[] = $uri->getNormalizedURI();
     }
 
     PhabricatorRepositoryURIIndex::updateRepositoryURIs(
       $this->getPHID(),
-      $uris);
+      $indexes);
 
     return $this;
   }
 
-  private function getNormalizedURI($uri) {
-    switch ($this->getVersionControlSystem()) {
-      case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
-        return new PhabricatorRepositoryURINormalizer(
-          PhabricatorRepositoryURINormalizer::TYPE_GIT,
-          $uri);
-      case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
-        return new PhabricatorRepositoryURINormalizer(
-          PhabricatorRepositoryURINormalizer::TYPE_SVN,
-          $uri);
-      case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
-        return new PhabricatorRepositoryURINormalizer(
-          PhabricatorRepositoryURINormalizer::TYPE_MERCURIAL,
-          $uri);
-      default:
-        throw new Exception(pht('Unrecognized version control system.'));
-    }
-  }
-
   public function isTracked() {
     $status = $this->getDetail('tracking-enabled');
     $map = self::getStatusMap();

src/applications/repository/storage/PhabricatorRepositoryURI.php

@@ -196,6 +196,26 @@ final class PhabricatorRepositoryURI
     return $this->getURIObject(false);
   }
 
+  public function getNormalizedURI() {
+    $vcs = $this->getRepository()->getVersionControlSystem();
+
+    $map = array(
+      PhabricatorRepositoryType::REPOSITORY_TYPE_GIT =>
+        PhabricatorRepositoryURINormalizer::TYPE_GIT,
+      PhabricatorRepositoryType::REPOSITORY_TYPE_SVN =>
+        PhabricatorRepositoryURINormalizer::TYPE_SVN,
+      PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL =>
+        PhabricatorRepositoryURINormalizer::TYPE_MERCURIAL,
+    );
+
+    $type = $map[$vcs];
+    $display = (string)$this->getDisplayURI();
+
+    $normal_uri = new PhabricatorRepositoryURINormalizer($type, $display);
+
+    return $normal_uri->getNormalizedURI();
+  }
+
   public function getEffectiveURI() {
     return $this->getURIObject(true);
   }
@@ -693,6 +713,7 @@ final class PhabricatorRepositoryURI
         'raw' => $this->getURI(),
         'display' => (string)$this->getDisplayURI(),
         'effective' => (string)$this->getEffectiveURI(),
+        'normalized' => (string)$this->getNormalizedURI(),
       ),
       'io' => array(
         'raw' => $this->getIOType(),