Add commits to the Phabricator search index

Summary:
create the indexer for commit

Test Plan:
run the reindex_one_commit.php against one existing commit
and it is returned in search result on the webpage; run parse_one_commit
against another commit; modified reindex_everything.php to let it only
parse one commit after loading all commits.

Reviewed By: epriestley
Reviewers: epriestley, aran
CC: aran, jungejason, epriestley, debow
Differential Revision: 490

scripts/search/index_one_commit.php

@@ -0,0 +1,53 @@
+#!/usr/bin/env php
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$root = dirname(dirname(dirname(__FILE__)));
+require_once $root.'/scripts/__init_script__.php';
+require_once $root.'/scripts/__init_env__.php';
+
+if (empty($argv[1])) {
+  echo "usage: index_one_commit.php <commit_name>\n";
+  die(1);
+}
+
+$commit = isset($argv[1]) ? $argv[1] : null;
+if (!$commit) {
+  throw new Exception("Provide a commit to index!");
+}
+$matches = null;
+if (!preg_match('/r([A-Z]+)([a-z0-9]+)/', $commit, $matches)) {
+  throw new Exception("Can't parse commit identifier!");
+}
+$repo = id(new PhabricatorRepository())->loadOneWhere(
+  'callsign = %s',
+  $matches[1]);
+if (!$repo) {
+  throw new Exception("Unknown repository!");
+}
+
+$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
+  'repositoryID = %d AND commitIdentifier = %s',
+  $repo->getID(),
+  $matches[2]);
+if (!$commit) {
+  throw new Exception('Unknown commit.');
+}
+
+PhabricatorSearchCommitIndexer::indexCommit($commit);
+echo "Done.\n";

scripts/search/reindex_everything.php

@@ -24,12 +24,6 @@ require_once $root.'/scripts/__init_env__.php';
 // TODO: Get rid of this script eventually, once this stuff is better-formalized
 // in Timeline consumers.
 
-phutil_require_module('phutil', 'symbols');
-PhutilSymbolLoader::loadClass('DifferentialRevision');
-PhutilSymbolLoader::loadClass('PhabricatorSearchDifferentialIndexer');
-PhutilSymbolLoader::loadClass('ManiphestTask');
-PhutilSymbolLoader::loadClass('PhabricatorSearchManiphestIndexer');
-
 echo "Loading revisions...\n";
 $revs = id(new DifferentialRevision())->loadAll();
 $count = count($revs);
@@ -40,6 +34,16 @@ foreach ($revs as $rev) {
 }
 echo "\n";
 
+echo "Loading commits...\n";
+$commits = id(new PhabricatorRepositoryCommit())->loadAll();
+$count = count($commits);
+echo "Reindexing {$count} commits";
+foreach ($commits as $commit) {
+  PhabricatorSearchCommitIndexer::indexCommit($commit);
+  echo '.';
+}
+echo "\n";
+
 echo "Loading tasks...\n";
 $tasks = id(new ManiphestTask())->loadAll();
 $count = count($tasks);

src/__phutil_library_map__.php

@@ -479,6 +479,7 @@ phutil_register_library_map(array(
     'PhabricatorSearchAbstractDocument' => 'applications/search/index/abstractdocument',
     'PhabricatorSearchAttachController' => 'applications/search/controller/attach',
     'PhabricatorSearchBaseController' => 'applications/search/controller/base',
+    'PhabricatorSearchCommitIndexer' => 'applications/search/index/indexer/repository',
     'PhabricatorSearchController' => 'applications/search/controller/search',
     'PhabricatorSearchDAO' => 'applications/search/storage/base',
     'PhabricatorSearchDifferentialIndexer' => 'applications/search/index/indexer/differential',
@@ -943,6 +944,7 @@ phutil_register_library_map(array(
     'PhabricatorRepositorySvnCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
     'PhabricatorSearchAttachController' => 'PhabricatorSearchController',
     'PhabricatorSearchBaseController' => 'PhabricatorController',
+    'PhabricatorSearchCommitIndexer' => 'PhabricatorSearchDocumentIndexer',
     'PhabricatorSearchController' => 'PhabricatorSearchBaseController',
     'PhabricatorSearchDAO' => 'PhabricatorLiskDAO',
     'PhabricatorSearchDifferentialIndexer' => 'PhabricatorSearchDocumentIndexer',

src/applications/repository/storage/commitdata/PhabricatorRepositoryCommitData.php

@@ -18,6 +18,8 @@
 
 class PhabricatorRepositoryCommitData extends PhabricatorRepositoryDAO {
 
+  const SUMMARY_MAX_LENGTH = 100;
+
   protected $commitID;
   protected $authorName;
   protected $commitMessage;
@@ -33,7 +35,12 @@ class PhabricatorRepositoryCommitData extends PhabricatorRepositoryDAO {
   }
 
   public function getSummary() {
-    return substr($this->getCommitMessage(), 0, 80);
+    $message = $this->getCommitMessage();
+    $lines = explode("\n", $message);
+    $summary = head($lines);
+    $summary = substr($summary, 0, self::SUMMARY_MAX_LENGTH);
+
+    return $summary;
   }
 
   public function getCommitDetail($key, $default = null) {

src/applications/repository/worker/commitchangeparser/base/PhabricatorRepositoryCommitChangeParserWorker.php

@@ -73,6 +73,8 @@ abstract class PhabricatorRepositoryCommitChangeParserWorker
 
   protected function finishParse() {
     $commit = $this->commit;
+    PhabricatorSearchCommitIndexer::indexCommit($commit);
+
     if ($this->shouldQueueFollowupTasks()) {
       $task = new PhabricatorWorkerTask();
       $task->setTaskClass('PhabricatorRepositoryCommitHeraldWorker');

src/applications/repository/worker/commitchangeparser/base/__init__.php

@@ -8,6 +8,7 @@
 
 phutil_require_module('phabricator', 'applications/repository/storage/repository');
 phutil_require_module('phabricator', 'applications/repository/worker/base');
+phutil_require_module('phabricator', 'applications/search/index/indexer/repository');
 phutil_require_module('phabricator', 'infrastructure/daemon/workers/storage/task');
 phutil_require_module('phabricator', 'storage/qsprintf');
 phutil_require_module('phabricator', 'storage/queryfx');

src/applications/search/constants/relationship/PhabricatorSearchRelationship.php

@@ -24,6 +24,7 @@ final class PhabricatorSearchRelationship {
   const RELATIONSHIP_COMMENTER  = 'comm';
   const RELATIONSHIP_OWNER      = 'ownr';
   const RELATIONSHIP_PROJECT    = 'proj';
+  const RELATIONSHIP_REPOSITORY = 'repo';
 
   const RELATIONSHIP_OPEN       = 'open';
   const RELATIONSHIP_TOUCH      = 'poke';

src/applications/search/controller/search/PhabricatorSearchController.php

@@ -70,6 +70,7 @@ class PhabricatorSearchController extends PhabricatorSearchBaseController {
     $options = array(
       '' => 'All Documents',
       PhabricatorPHIDConstants::PHID_TYPE_DREV => 'Differential Revisions',
+      PhabricatorPHIDConstants::PHID_TYPE_CMIT => 'Repository Commits',
       PhabricatorPHIDConstants::PHID_TYPE_TASK => 'Maniphest Tasks',
     ) + $more;
 

src/applications/search/execute/mysql/PhabricatorSearchMySQLExecutor.php

@@ -165,6 +165,12 @@ class PhabricatorSearchMySQLExecutor extends PhabricatorSearchExecutor {
       'project',
       PhabricatorSearchRelationship::RELATIONSHIP_PROJECT);
 
+    $join[] = $this->joinRelationship(
+      $conn_r,
+      $query,
+      'repository',
+      PhabricatorSearchRelationship::RELATIONSHIP_REPOSITORY);
+
 /*
     $join[] = $this->joinRelationship(
       $conn_r,

src/applications/search/index/indexer/repository/PhabricatorSearchCommitIndexer.php

@@ -0,0 +1,65 @@
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+class PhabricatorSearchCommitIndexer
+  extends PhabricatorSearchDocumentIndexer {
+
+  public static function indexCommit(PhabricatorRepositoryCommit $commit) {
+    $commit_data = id(new PhabricatorRepositoryCommitData())->loadOneWhere(
+      'commitID = %d',
+      $commit->getID());
+    $date_created = $commit->getEpoch();
+    $commit_message = $commit_data->getCommitMessage();
+    $author_phid = $commit_data->getCommitDetail('authorPHID');
+
+    $repository = id(new PhabricatorRepository())->loadOneWhere(
+      'id = %d',
+      $commit->getRepositoryID());
+
+    $title = 'r'.$repository->getCallsign().$commit->getCommitIdentifier().
+      " ".$commit_data->getSummary();
+
+    $doc = new PhabricatorSearchAbstractDocument();
+    $doc->setPHID($commit->getPHID());
+    $doc->setDocumentType(PhabricatorPHIDConstants::PHID_TYPE_CMIT);
+    $doc->setDocumentCreated($date_created);
+    $doc->setDocumentModified($date_created);
+    $doc->setDocumentTitle($title);
+
+    $doc->addField(
+      PhabricatorSearchField::FIELD_BODY,
+      $commit_message);
+
+    if ($author_phid) {
+      $doc->addRelationship(
+        PhabricatorSearchRelationship::RELATIONSHIP_AUTHOR,
+        $author_phid,
+        PhabricatorPHIDConstants::PHID_TYPE_USER,
+        $date_created);
+    }
+
+    $doc->addRelationship(
+      PhabricatorSearchRelationship::RELATIONSHIP_REPOSITORY,
+      $repository->getPHID(),
+      PhabricatorPHIDConstants::PHID_TYPE_REPO,
+      $date_created);
+
+    PhabricatorSearchDocument::reindexAbstractDocument($doc);
+  }
+}
+

src/applications/search/index/indexer/repository/__init__.php

@@ -0,0 +1,21 @@
+<?php
+/**
+ * This file is automatically generated. Lint this module to rebuild it.
+ * @generated
+ */
+
+
+
+phutil_require_module('phabricator', 'applications/phid/constants');
+phutil_require_module('phabricator', 'applications/repository/storage/commitdata');
+phutil_require_module('phabricator', 'applications/repository/storage/repository');
+phutil_require_module('phabricator', 'applications/search/constants/field');
+phutil_require_module('phabricator', 'applications/search/constants/relationship');
+phutil_require_module('phabricator', 'applications/search/index/abstractdocument');
+phutil_require_module('phabricator', 'applications/search/index/indexer/base');
+phutil_require_module('phabricator', 'applications/search/storage/document/document');
+
+phutil_require_module('phutil', 'utils');
+
+
+phutil_require_source('PhabricatorSearchCommitIndexer.php');