From ff3333548f8abc1c21707387639fa215af20be2b Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Thu, 24 Nov 2016 08:36:32 -0800
Subject: [PATCH] Create and populate a stopwords table for InnoDB fulltext
 indexes to use in the future

Summary:
Ref T11741. InnoDB uses a stopwords table instead of a stopwords file.

During `storage upgrade`, synchronize the table from the stopwords file on disk.

Test Plan:
  - Ran `storage upgrade`.
  - Ran `select * from stopwords`, saw stopwords.
  - Added some garbage to the table.
  - Ran `storage upgrade`, saw it remove it.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T11741

Differential Revision: https://secure.phabricator.com/D16940
---
 .../20161124.search.01.stopwords.sql          |  3 ++
 .../storage/PhabricatorSearchSchemaSpec.php   |  8 +++
 .../document/PhabricatorSearchDocument.php    |  2 +
 ...icatorStorageManagementUpgradeWorkflow.php | 53 +++++++++++++++++++
 4 files changed, 66 insertions(+)
 create mode 100644 resources/sql/autopatches/20161124.search.01.stopwords.sql

diff --git a/resources/sql/autopatches/20161124.search.01.stopwords.sql b/resources/sql/autopatches/20161124.search.01.stopwords.sql
new file mode 100644
index 0000000000..8cbf2dc50b
--- /dev/null
+++ b/resources/sql/autopatches/20161124.search.01.stopwords.sql
@@ -0,0 +1,3 @@
+CREATE TABLE {$NAMESPACE}_search.stopwords (
+  value VARCHAR(32) NOT NULL COLLATE {$COLLATE_SORT}
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};
diff --git a/src/applications/search/storage/PhabricatorSearchSchemaSpec.php b/src/applications/search/storage/PhabricatorSearchSchemaSpec.php
index 4e9e739016..10d16e60b5 100644
--- a/src/applications/search/storage/PhabricatorSearchSchemaSpec.php
+++ b/src/applications/search/storage/PhabricatorSearchSchemaSpec.php
@@ -5,6 +5,14 @@ final class PhabricatorSearchSchemaSpec
 
   public function buildSchemata() {
     $this->buildEdgeSchemata(new PhabricatorProfilePanelConfiguration());
+
+    $this->buildRawSchema(
+      'search',
+      PhabricatorSearchDocument::STOPWORDS_TABLE,
+      array(
+        'value' => 'sort32',
+      ),
+      array());
   }
 
 }
diff --git a/src/applications/search/storage/document/PhabricatorSearchDocument.php b/src/applications/search/storage/document/PhabricatorSearchDocument.php
index 28080944b3..1016872572 100644
--- a/src/applications/search/storage/document/PhabricatorSearchDocument.php
+++ b/src/applications/search/storage/document/PhabricatorSearchDocument.php
@@ -7,6 +7,8 @@ final class PhabricatorSearchDocument extends PhabricatorSearchDAO {
   protected $documentCreated;
   protected $documentModified;
 
+  const STOPWORDS_TABLE = 'stopwords';
+
   protected function getConfiguration() {
     return array(
       self::CONFIG_TIMESTAMPS => false,
diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php
index b4e6e280a5..e0b8d884c5 100644
--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php
+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php
@@ -77,6 +77,17 @@ final class PhabricatorStorageManagementUpgradeWorkflow
 
     $this->upgradeSchemata($apis, $apply_only, $no_quickstart, $init_only);
 
+    if ($apply_only || $init_only) {
+      echo tsprintf(
+        "%s\n",
+        pht('Declining to synchronize static tables.'));
+    } else {
+      echo tsprintf(
+        "%s\n",
+        pht('Synchronizing static tables...'));
+      $this->synchronizeSchemata();
+    }
+
     if ($no_adjust || $init_only || $apply_only) {
       $console->writeOut(
         "%s\n",
@@ -93,4 +104,46 @@ final class PhabricatorStorageManagementUpgradeWorkflow
     return 0;
   }
 
+  private function synchronizeSchemata() {
+    // Synchronize the InnoDB fulltext stopwords table from the stopwords file
+    // on disk.
+
+    $table = new PhabricatorSearchDocument();
+    $conn = $table->establishConnection('w');
+    $table_ref = PhabricatorSearchDocument::STOPWORDS_TABLE;
+
+    $stopwords_database = queryfx_all(
+      $conn,
+      'SELECT value FROM %T',
+      $table_ref);
+    $stopwords_database = ipull($stopwords_database, 'value', 'value');
+
+    $stopwords_path = phutil_get_library_root('phabricator');
+    $stopwords_path = $stopwords_path.'/../resources/sql/stopwords.txt';
+    $stopwords_file = Filesystem::readFile($stopwords_path);
+    $stopwords_file = phutil_split_lines($stopwords_file, false);
+    $stopwords_file = array_fuse($stopwords_file);
+
+    $rem_words = array_diff_key($stopwords_database, $stopwords_file);
+    if ($rem_words) {
+      queryfx(
+        $conn,
+        'DELETE FROM %T WHERE value IN (%Ls)',
+        $table_ref,
+        $rem_words);
+    }
+
+    $add_words = array_diff_key($stopwords_file, $stopwords_database);
+    if ($add_words) {
+      foreach ($add_words as $word) {
+        queryfx(
+          $conn,
+          'INSERT IGNORE INTO %T (value) VALUES (%s)',
+          $table_ref,
+          $word);
+      }
+    }
+  }
+
+
 }