diff --git a/resources/sql/autopatches/20171002.cngram.01.maniphest.sql b/resources/sql/autopatches/20171002.cngram.01.maniphest.sql new file mode 100644 index 0000000000..9b275f5b45 --- /dev/null +++ b/resources/sql/autopatches/20171002.cngram.01.maniphest.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_maniphest.maniphest_task_fngrams_common ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}, + needsCollection BOOL NOT NULL, + UNIQUE KEY `key_ngram` (ngram), + KEY `key_collect` (needsCollection) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20171002.cngram.02.event.sql b/resources/sql/autopatches/20171002.cngram.02.event.sql new file mode 100644 index 0000000000..a071fdcd19 --- /dev/null +++ b/resources/sql/autopatches/20171002.cngram.02.event.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_calendar.calendar_event_fngrams_common ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}, + needsCollection BOOL NOT NULL, + UNIQUE KEY `key_ngram` (ngram), + KEY `key_collect` (needsCollection) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20171002.cngram.03.revision.sql b/resources/sql/autopatches/20171002.cngram.03.revision.sql new file mode 100644 index 0000000000..40c2450598 --- /dev/null +++ b/resources/sql/autopatches/20171002.cngram.03.revision.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_differential.differential_revision_fngrams_common ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}, + needsCollection BOOL NOT NULL, + UNIQUE KEY `key_ngram` (ngram), + KEY `key_collect` (needsCollection) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20171002.cngram.04.fund.sql b/resources/sql/autopatches/20171002.cngram.04.fund.sql new file mode 100644 index 0000000000..34975ce4fb --- /dev/null +++ b/resources/sql/autopatches/20171002.cngram.04.fund.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_fund.fund_initiative_fngrams_common ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}, + needsCollection BOOL NOT NULL, + UNIQUE KEY `key_ngram` (ngram), + KEY `key_collect` (needsCollection) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20171002.cngram.05.owners.sql b/resources/sql/autopatches/20171002.cngram.05.owners.sql new file mode 100644 index 0000000000..e98d29f87c --- /dev/null +++ b/resources/sql/autopatches/20171002.cngram.05.owners.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_owners.owners_package_fngrams_common ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}, + needsCollection BOOL NOT NULL, + UNIQUE KEY `key_ngram` (ngram), + KEY `key_collect` (needsCollection) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20171002.cngram.06.passphrase.sql b/resources/sql/autopatches/20171002.cngram.06.passphrase.sql new file mode 100644 index 0000000000..f9afa9ad87 --- /dev/null +++ b/resources/sql/autopatches/20171002.cngram.06.passphrase.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_passphrase.passphrase_credential_fngrams_common ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}, + needsCollection BOOL NOT NULL, + UNIQUE KEY `key_ngram` (ngram), + KEY `key_collect` (needsCollection) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20171002.cngram.07.blog.sql b/resources/sql/autopatches/20171002.cngram.07.blog.sql new file mode 100644 index 0000000000..34001c3608 --- /dev/null +++ b/resources/sql/autopatches/20171002.cngram.07.blog.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_phame.phame_blog_fngrams_common ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}, + needsCollection BOOL NOT NULL, + UNIQUE KEY `key_ngram` (ngram), + KEY `key_collect` (needsCollection) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20171002.cngram.08.post.sql b/resources/sql/autopatches/20171002.cngram.08.post.sql new file mode 100644 index 0000000000..9a9c70867e --- /dev/null +++ b/resources/sql/autopatches/20171002.cngram.08.post.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_phame.phame_post_fngrams_common ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}, + needsCollection BOOL NOT NULL, + UNIQUE KEY `key_ngram` (ngram), + KEY `key_collect` (needsCollection) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20171002.cngram.09.pholio.sql b/resources/sql/autopatches/20171002.cngram.09.pholio.sql new file mode 100644 index 0000000000..6e8b8f8dcc --- /dev/null +++ b/resources/sql/autopatches/20171002.cngram.09.pholio.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_pholio.pholio_mock_fngrams_common ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}, + needsCollection BOOL NOT NULL, + UNIQUE KEY `key_ngram` (ngram), + KEY `key_collect` (needsCollection) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20171002.cngram.10.phriction.sql b/resources/sql/autopatches/20171002.cngram.10.phriction.sql new file mode 100644 index 0000000000..ed31dc30ba --- /dev/null +++ b/resources/sql/autopatches/20171002.cngram.10.phriction.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_phriction.phriction_document_fngrams_common ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}, + needsCollection BOOL NOT NULL, + UNIQUE KEY `key_ngram` (ngram), + KEY `key_collect` (needsCollection) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20171002.cngram.11.project.sql b/resources/sql/autopatches/20171002.cngram.11.project.sql new file mode 100644 index 0000000000..9c11235ba7 --- /dev/null +++ b/resources/sql/autopatches/20171002.cngram.11.project.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_project.project_project_fngrams_common ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}, + needsCollection BOOL NOT NULL, + UNIQUE KEY `key_ngram` (ngram), + KEY `key_collect` (needsCollection) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20171002.cngram.12.user.sql b/resources/sql/autopatches/20171002.cngram.12.user.sql new file mode 100644 index 0000000000..3e8499aaa6 --- /dev/null +++ b/resources/sql/autopatches/20171002.cngram.12.user.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_user.user_user_fngrams_common ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}, + needsCollection BOOL NOT NULL, + UNIQUE KEY `key_ngram` (ngram), + KEY `key_collect` (needsCollection) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20171002.cngram.13.repository.sql b/resources/sql/autopatches/20171002.cngram.13.repository.sql new file mode 100644 index 0000000000..e406c44edf --- /dev/null +++ b/resources/sql/autopatches/20171002.cngram.13.repository.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_repository.repository_repository_fngrams_common ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}, + needsCollection BOOL NOT NULL, + UNIQUE KEY `key_ngram` (ngram), + KEY `key_collect` (needsCollection) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20171002.cngram.14.commit.sql b/resources/sql/autopatches/20171002.cngram.14.commit.sql new file mode 100644 index 0000000000..48c1a02594 --- /dev/null +++ b/resources/sql/autopatches/20171002.cngram.14.commit.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_repository.repository_commit_fngrams_common ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}, + needsCollection BOOL NOT NULL, + UNIQUE KEY `key_ngram` (ngram), + KEY `key_collect` (needsCollection) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index c535d46322..a650e33fcf 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3941,6 +3941,7 @@ phutil_register_library_map(array( 'PhabricatorSearchEngineAttachment' => 'applications/search/engineextension/PhabricatorSearchEngineAttachment.php', 'PhabricatorSearchEngineExtension' => 'applications/search/engineextension/PhabricatorSearchEngineExtension.php', 'PhabricatorSearchEngineExtensionModule' => 'applications/search/engineextension/PhabricatorSearchEngineExtensionModule.php', + 'PhabricatorSearchFerretNgramGarbageCollector' => 'applications/search/garbagecollector/PhabricatorSearchFerretNgramGarbageCollector.php', 'PhabricatorSearchField' => 'applications/search/field/PhabricatorSearchField.php', 'PhabricatorSearchHost' => 'infrastructure/cluster/search/PhabricatorSearchHost.php', 'PhabricatorSearchHovercardController' => 'applications/search/controller/PhabricatorSearchHovercardController.php', @@ -3948,6 +3949,8 @@ phutil_register_library_map(array( 'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'applications/search/engineextension/PhabricatorSearchIndexVersionDestructionEngineExtension.php', 'PhabricatorSearchManagementIndexWorkflow' => 'applications/search/management/PhabricatorSearchManagementIndexWorkflow.php', 'PhabricatorSearchManagementInitWorkflow' => 'applications/search/management/PhabricatorSearchManagementInitWorkflow.php', + 'PhabricatorSearchManagementNgramsWorkflow' => 'applications/search/management/PhabricatorSearchManagementNgramsWorkflow.php', + 'PhabricatorSearchManagementQueryWorkflow' => 'applications/search/management/PhabricatorSearchManagementQueryWorkflow.php', 'PhabricatorSearchManagementWorkflow' => 'applications/search/management/PhabricatorSearchManagementWorkflow.php', 'PhabricatorSearchNgrams' => 'applications/search/ngrams/PhabricatorSearchNgrams.php', 'PhabricatorSearchNgramsDestructionEngineExtension' => 'applications/search/engineextension/PhabricatorSearchNgramsDestructionEngineExtension.php', @@ -9521,6 +9524,7 @@ phutil_register_library_map(array( 'PhabricatorSearchEngineAttachment' => 'Phobject', 'PhabricatorSearchEngineExtension' => 'Phobject', 'PhabricatorSearchEngineExtensionModule' => 'PhabricatorConfigModule', + 'PhabricatorSearchFerretNgramGarbageCollector' => 'PhabricatorGarbageCollector', 'PhabricatorSearchField' => 'Phobject', 'PhabricatorSearchHost' => 'Phobject', 'PhabricatorSearchHovercardController' => 'PhabricatorSearchBaseController', @@ -9528,6 +9532,8 @@ phutil_register_library_map(array( 'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'PhabricatorDestructionEngineExtension', 'PhabricatorSearchManagementIndexWorkflow' => 'PhabricatorSearchManagementWorkflow', 'PhabricatorSearchManagementInitWorkflow' => 'PhabricatorSearchManagementWorkflow', + 'PhabricatorSearchManagementNgramsWorkflow' => 'PhabricatorSearchManagementWorkflow', + 'PhabricatorSearchManagementQueryWorkflow' => 'PhabricatorSearchManagementWorkflow', 'PhabricatorSearchManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorSearchNgrams' => 'PhabricatorSearchDAO', 'PhabricatorSearchNgramsDestructionEngineExtension' => 'PhabricatorDestructionEngineExtension', diff --git a/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php b/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php index 97d6e1ac81..c9e78af0bc 100644 --- a/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php +++ b/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php @@ -30,6 +30,9 @@ final class PhabricatorCacheSchemaSpec extends PhabricatorConfigSchemaSpec { 'key_ttl' => array( 'columns' => array('cacheExpires'), ), + ), + array( + 'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_CACHE, )); } diff --git a/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php b/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php index a67c57e6f9..760317ae80 100644 --- a/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php +++ b/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php @@ -261,6 +261,7 @@ final class PhabricatorConfigDatabaseStatusController $this->renderAttr( $table->getCollation(), $table->hasIssue($collation_issue)), + $table->getPersistenceTypeDisplayName(), ); } @@ -270,12 +271,14 @@ final class PhabricatorConfigDatabaseStatusController null, pht('Table'), pht('Collation'), + pht('Persistence'), )) ->setColumnClasses( array( null, 'wide pri', null, + null, )); $title = $database_name; diff --git a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php index 6feda7363c..2cb7763110 100644 --- a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php +++ b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php @@ -338,6 +338,8 @@ final class PhabricatorConfigSchemaQuery extends Phobject { $comp_table->addKey($comp_key); } + $comp_table->setPersistenceType($expect_table->getPersistenceType()); + $comp_database->addTable($comp_table); } $comp_server->addDatabase($comp_database); diff --git a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php index b24adf27cd..c451658682 100644 --- a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php +++ b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php @@ -56,30 +56,52 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject { } protected function buildFerretIndexSchema(PhabricatorFerretEngine $engine) { + $index_options = array( + 'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_INDEX, + ); + $this->buildRawSchema( $engine->getApplicationName(), $engine->getDocumentTableName(), $engine->getDocumentSchemaColumns(), - $engine->getDocumentSchemaKeys()); + $engine->getDocumentSchemaKeys(), + $index_options); $this->buildRawSchema( $engine->getApplicationName(), $engine->getFieldTableName(), $engine->getFieldSchemaColumns(), - $engine->getFieldSchemaKeys()); + $engine->getFieldSchemaKeys(), + $index_options); $this->buildRawSchema( $engine->getApplicationName(), $engine->getNgramsTableName(), $engine->getNgramsSchemaColumns(), - $engine->getNgramsSchemaKeys()); + $engine->getNgramsSchemaKeys(), + $index_options); + + $this->buildRawSchema( + $engine->getApplicationName(), + $engine->getCommonNgramsTableName(), + $engine->getCommonNgramsSchemaColumns(), + $engine->getCommonNgramsSchemaKeys(), + $index_options); } protected function buildRawSchema( $database_name, $table_name, array $columns, - array $keys) { + array $keys, + array $options = array()) { + + PhutilTypeSpec::checkMap( + $options, + array( + 'persistence' => 'optional string', + )); + $database = $this->getDatabase($database_name); $table = $this->newTable($table_name); @@ -138,6 +160,11 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject { $table->addKey($key); } + $persistence_type = idx($options, 'persistence'); + if ($persistence_type !== null) { + $table->setPersistenceType($persistence_type); + } + $database->addTable($table); } diff --git a/src/applications/config/schema/PhabricatorConfigTableSchema.php b/src/applications/config/schema/PhabricatorConfigTableSchema.php index 870c8ebddb..d6e5bfd9fa 100644 --- a/src/applications/config/schema/PhabricatorConfigTableSchema.php +++ b/src/applications/config/schema/PhabricatorConfigTableSchema.php @@ -7,6 +7,11 @@ final class PhabricatorConfigTableSchema private $engine; private $columns = array(); private $keys = array(); + private $persistenceType = self::PERSISTENCE_DATA; + + const PERSISTENCE_DATA = 'data'; + const PERSISTENCE_CACHE = 'cache'; + const PERSISTENCE_INDEX = 'index'; public function addColumn(PhabricatorConfigColumnSchema $column) { $key = $column->getName(); @@ -45,6 +50,27 @@ final class PhabricatorConfigTableSchema return idx($this->getKeys(), $key); } + public function setPersistenceType($persistence_type) { + $this->persistenceType = $persistence_type; + return $this; + } + + public function getPersistenceType() { + return $this->persistenceType; + } + + public function getPersistenceTypeDisplayName() { + $map = array( + self::PERSISTENCE_DATA => pht('Data'), + self::PERSISTENCE_CACHE => pht('Cache'), + self::PERSISTENCE_INDEX => pht('Index'), + ); + + $type = $this->getPersistenceType(); + + return idx($map, $type, $type); + } + protected function getSubschemata() { // NOTE: Keys and columns may have the same name, so make sure we return // everything. diff --git a/src/applications/differential/storage/DifferentialSchemaSpec.php b/src/applications/differential/storage/DifferentialSchemaSpec.php index e376c7f4f8..7aa76fa822 100644 --- a/src/applications/differential/storage/DifferentialSchemaSpec.php +++ b/src/applications/differential/storage/DifferentialSchemaSpec.php @@ -21,6 +21,9 @@ final class DifferentialSchemaSpec extends PhabricatorConfigSchemaSpec { 'dateCreated' => array( 'columns' => array('dateCreated'), ), + ), + array( + 'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_CACHE, )); $this->buildRawSchema( diff --git a/src/applications/diffusion/application/PhabricatorDiffusionApplication.php b/src/applications/diffusion/application/PhabricatorDiffusionApplication.php index cf9c14aa8b..5a426ec299 100644 --- a/src/applications/diffusion/application/PhabricatorDiffusionApplication.php +++ b/src/applications/diffusion/application/PhabricatorDiffusionApplication.php @@ -69,10 +69,11 @@ final class PhabricatorDiffusionApplication extends PhabricatorApplication { 'branches/(?P.*)' => 'DiffusionBranchTableController', 'refs/(?P.*)' => 'DiffusionRefTableController', 'lint/(?P.*)' => 'DiffusionLintController', - 'commit/(?P[a-z0-9]+)/branches/' - => 'DiffusionCommitBranchesController', - 'commit/(?P[a-z0-9]+)/tags/' - => 'DiffusionCommitTagsController', + 'commit/(?P[a-z0-9]+)' => array( + '/?' => 'DiffusionCommitController', + '/branches/' => 'DiffusionCommitBranchesController', + '/tags/' => 'DiffusionCommitTagsController', + ), 'compare/' => 'DiffusionCompareController', 'manage/(?:(?P[^/]+)/)?' => 'DiffusionRepositoryManagePanelsController', diff --git a/src/applications/diffusion/conduit/DiffusionLastModifiedQueryConduitAPIMethod.php b/src/applications/diffusion/conduit/DiffusionLastModifiedQueryConduitAPIMethod.php index 30b436cfff..1135420a6e 100644 --- a/src/applications/diffusion/conduit/DiffusionLastModifiedQueryConduitAPIMethod.php +++ b/src/applications/diffusion/conduit/DiffusionLastModifiedQueryConduitAPIMethod.php @@ -115,6 +115,10 @@ final class DiffusionLastModifiedQueryConduitAPIMethod $graph_cache = new PhabricatorRepositoryGraphCache(); $results = array(); + + // Spend no more than this many total seconds trying to satisfy queries + // via the graph cache. + $remaining_time = 10.0; foreach ($map as $path => $commit) { $path_id = idx($path_map, $path); if (!$path_id) { @@ -125,13 +129,21 @@ final class DiffusionLastModifiedQueryConduitAPIMethod continue; } + $t_start = microtime(true); $cache_result = $graph_cache->loadLastModifiedCommitID( $commit_id, - $path_id); + $path_id, + $remaining_time); + $t_end = microtime(true); if ($cache_result !== false) { $results[$path] = $cache_result; } + + $remaining_time -= ($t_end - $t_start); + if ($remaining_time <= 0) { + break; + } } if ($results) { diff --git a/src/applications/diffusion/controller/DiffusionCommitController.php b/src/applications/diffusion/controller/DiffusionCommitController.php index 59a132b17d..38c94e8a92 100644 --- a/src/applications/diffusion/controller/DiffusionCommitController.php +++ b/src/applications/diffusion/controller/DiffusionCommitController.php @@ -22,17 +22,27 @@ final class DiffusionCommitController extends DiffusionController { $drequest = $this->getDiffusionRequest(); $viewer = $request->getUser(); + $repository = $drequest->getRepository(); + $commit_identifier = $drequest->getCommit(); + + // If this page is being accessed via "/source/xyz/commit/...", redirect + // to the canonical URI. + $has_callsign = strlen($request->getURIData('repositoryCallsign')); + $has_id = strlen($request->getURIData('repositoryID')); + if (!$has_callsign && !$has_id) { + $canonical_uri = $repository->getCommitURI($commit_identifier); + return id(new AphrontRedirectResponse()) + ->setURI($canonical_uri); + } if ($request->getStr('diff')) { return $this->buildRawDiffResponse($drequest); } - $repository = $drequest->getRepository(); - $commit = id(new DiffusionCommitQuery()) ->setViewer($viewer) ->withRepository($repository) - ->withIdentifiers(array($drequest->getCommit())) + ->withIdentifiers(array($commit_identifier)) ->needCommitData(true) ->needAuditRequests(true) ->executeOne(); diff --git a/src/applications/repository/graphcache/PhabricatorRepositoryGraphCache.php b/src/applications/repository/graphcache/PhabricatorRepositoryGraphCache.php index d52182fcc9..c4adc61ab0 100644 --- a/src/applications/repository/graphcache/PhabricatorRepositoryGraphCache.php +++ b/src/applications/repository/graphcache/PhabricatorRepositoryGraphCache.php @@ -102,6 +102,10 @@ final class PhabricatorRepositoryGraphCache extends Phobject { } // Otherwise, the rebuild gave us the data, so we can keep going. + + $did_fill = true; + } else { + $did_fill = false; } // Sanity check so we can survive and recover from bad data. @@ -147,12 +151,17 @@ final class PhabricatorRepositoryGraphCache extends Phobject { $commit_id = $parent_id; // Periodically check if we've spent too long looking for a result - // in the cache, and return so we can fall back to a VCS operation. This - // keeps us from having a degenerate worst case if, e.g., the cache - // is cold and we need to inspect a very large number of blocks + // in the cache, and return so we can fall back to a VCS operation. + // This keeps us from having a degenerate worst case if, e.g., the + // cache is cold and we need to inspect a very large number of blocks // to satisfy the query. - if (((++$iterations) % 64) === 0) { + ++$iterations; + + // If we performed a cache fill in this cycle, always check the time + // limit, since cache fills may take a significant amount of time. + + if ($did_fill || ($iterations % 64 === 0)) { $t_end = microtime(true); if (($t_end - $t_start) > $time) { return false; diff --git a/src/applications/search/engineextension/PhabricatorFerretFulltextEngineExtension.php b/src/applications/search/engineextension/PhabricatorFerretFulltextEngineExtension.php index 6903072345..fe50518f3f 100644 --- a/src/applications/search/engineextension/PhabricatorFerretFulltextEngineExtension.php +++ b/src/applications/search/engineextension/PhabricatorFerretFulltextEngineExtension.php @@ -165,21 +165,46 @@ final class PhabricatorFerretFulltextEngineExtension $ferret_field['normalCorpus']); } - $sql = array(); - foreach ($ngrams as $ngram) { - $sql[] = qsprintf( + if ($ngrams) { + $common = queryfx_all( $conn, - '(%d, %s)', - $document_id, - $ngram); + 'SELECT ngram FROM %T WHERE ngram IN (%Ls)', + $engine->getCommonNgramsTableName(), + $ngrams); + $common = ipull($common, 'ngram', 'ngram'); + + foreach ($ngrams as $key => $ngram) { + if (isset($common[$ngram])) { + unset($ngrams[$key]); + continue; + } + + // NOTE: MySQL discards trailing whitespace in CHAR(X) columns. + $trim_ngram = rtrim($ngram, ' '); + if (isset($common[$ngram])) { + unset($ngrams[$key]); + continue; + } + } } - foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) { - queryfx( - $conn, - 'INSERT INTO %T (documentID, ngram) VALUES %Q', - $engine->getNgramsTableName(), - $chunk); + if ($ngrams) { + $sql = array(); + foreach ($ngrams as $ngram) { + $sql[] = qsprintf( + $conn, + '(%d, %s)', + $document_id, + $ngram); + } + + foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) { + queryfx( + $conn, + 'INSERT INTO %T (documentID, ngram) VALUES %Q', + $engine->getNgramsTableName(), + $chunk); + } } } catch (Exception $ex) { $object->killTransaction(); diff --git a/src/applications/search/ferret/PhabricatorFerretEngine.php b/src/applications/search/ferret/PhabricatorFerretEngine.php index 7cd06ae549..ee6ca57ec3 100644 --- a/src/applications/search/ferret/PhabricatorFerretEngine.php +++ b/src/applications/search/ferret/PhabricatorFerretEngine.php @@ -295,4 +295,35 @@ abstract class PhabricatorFerretEngine extends Phobject { ); } + public function getCommonNgramsTableName() { + $application = $this->getApplicationName(); + $scope = $this->getScopeName(); + + return "{$application}_{$scope}_fngrams_common"; + } + + public function getCommonNgramsSchemaColumns() { + return array( + 'id' => 'auto', + 'ngram' => 'char3', + 'needsCollection' => 'bool', + ); + } + + public function getCommonNgramsSchemaKeys() { + return array( + 'PRIMARY' => array( + 'columns' => array('id'), + 'unique' => true, + ), + 'key_ngram' => array( + 'columns' => array('ngram'), + 'unique' => true, + ), + 'key_collect' => array( + 'columns' => array('needsCollection'), + ), + ); + } + } diff --git a/src/applications/search/garbagecollector/PhabricatorSearchFerretNgramGarbageCollector.php b/src/applications/search/garbagecollector/PhabricatorSearchFerretNgramGarbageCollector.php new file mode 100644 index 0000000000..f2c43743f6 --- /dev/null +++ b/src/applications/search/garbagecollector/PhabricatorSearchFerretNgramGarbageCollector.php @@ -0,0 +1,55 @@ +setAncestorClass('PhabricatorFerretInterface') + ->execute(); + + $did_collect = false; + foreach ($all_objects as $object) { + $engine = $object->newFerretEngine(); + $conn = $object->establishConnection('w'); + + $ngram_row = queryfx_one( + $conn, + 'SELECT ngram FROM %T WHERE needsCollection = 1 LIMIT 1', + $engine->getCommonNgramsTableName()); + if (!$ngram_row) { + continue; + } + + $ngram = $ngram_row['ngram']; + + queryfx( + $conn, + 'DELETE FROM %T WHERE ngram = %s', + $engine->getNgramsTableName(), + $ngram); + + queryfx( + $conn, + 'UPDATE %T SET needsCollection = 0 WHERE ngram = %s', + $engine->getCommonNgramsTableName(), + $ngram); + + $did_collect = true; + break; + } + + return $did_collect; + } + +} diff --git a/src/applications/search/management/PhabricatorSearchManagementNgramsWorkflow.php b/src/applications/search/management/PhabricatorSearchManagementNgramsWorkflow.php new file mode 100644 index 0000000000..4b983fe0f9 --- /dev/null +++ b/src/applications/search/management/PhabricatorSearchManagementNgramsWorkflow.php @@ -0,0 +1,109 @@ +setName('ngrams') + ->setSynopsis( + pht( + 'Recompute common ngrams. This is an advanced workflow that '. + 'can harm search quality if used improperly.')) + ->setArguments( + array( + array( + 'name' => 'reset', + 'help' => pht('Reset all common ngram records.'), + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $is_reset = $args->getArg('reset'); + + $all_objects = id(new PhutilClassMapQuery()) + ->setAncestorClass('PhabricatorFerretInterface') + ->execute(); + + $min_documents = 4096; + $threshold = 0.15; + + foreach ($all_objects as $object) { + $engine = $object->newFerretEngine(); + $conn = $object->establishConnection('w'); + $display_name = get_class($object); + + if ($is_reset) { + echo tsprintf( + "%s\n", + pht( + 'Resetting common ngrams for "%s".', + $display_name)); + + queryfx( + $conn, + 'DELETE FROM %T', + $engine->getCommonNgramsTableName()); + continue; + } + + $document_count = queryfx_one( + $conn, + 'SELECT COUNT(*) N FROM %T', + $engine->getDocumentTableName()); + $document_count = $document_count['N']; + + if ($document_count < $min_documents) { + echo tsprintf( + "%s\n", + pht( + 'Too few documents of type "%s" for any ngrams to be common.', + $display_name)); + continue; + } + + $min_frequency = (int)ceil($document_count * $threshold); + $common_ngrams = queryfx_all( + $conn, + 'SELECT ngram, COUNT(*) N FROM %T + GROUP BY ngram + HAVING N >= %d', + $engine->getNgramsTableName(), + $min_frequency); + + if (!$common_ngrams) { + echo tsprintf( + "%s\n", + pht( + 'No new common ngrams exist for "%s".', + $display_name)); + continue; + } + + $sql = array(); + foreach ($common_ngrams as $ngram) { + $sql[] = qsprintf( + $conn, + '(%s, 1)', + $ngram['ngram']); + } + + foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) { + queryfx( + $conn, + 'INSERT IGNORE INTO %T (ngram, needsCollection) + VALUES %Q', + $engine->getCommonNgramsTableName(), + $chunk); + } + + echo tsprintf( + "%s\n", + pht( + 'Updated common ngrams for "%s".', + $display_name)); + } + } + +} diff --git a/src/applications/search/management/PhabricatorSearchManagementQueryWorkflow.php b/src/applications/search/management/PhabricatorSearchManagementQueryWorkflow.php new file mode 100644 index 0000000000..8e40162a98 --- /dev/null +++ b/src/applications/search/management/PhabricatorSearchManagementQueryWorkflow.php @@ -0,0 +1,55 @@ +setName('query') + ->setSynopsis( + pht('Run a search query. Intended for debugging and development.')) + ->setArguments( + array( + array( + 'name' => 'query', + 'param' => 'query', + 'help' => pht('Raw query to execute.'), + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $viewer = $this->getViewer(); + $raw_query = $args->getArg('query'); + if (!strlen($raw_query)) { + throw new PhutilArgumentUsageException( + pht('Specify a query with --query.')); + } + + $engine = id(new PhabricatorSearchApplicationSearchEngine()) + ->setViewer($viewer); + + $saved = $engine->newSavedQuery(); + $saved->setParameter('query', $raw_query); + + $query = $engine->buildQueryFromSavedQuery($saved); + $pager = $engine->newPagerForSavedQuery($saved); + + $results = $engine->executeQuery($query, $pager); + if ($results) { + foreach ($results as $result) { + echo tsprintf( + "%s\t%s\n", + $result->getPHID(), + $result->getName()); + } + } else { + echo tsprintf( + "%s\n", + pht('No results.')); + } + + return 0; + } + +} diff --git a/src/applications/settings/panel/PhabricatorEmailAddressesSettingsPanel.php b/src/applications/settings/panel/PhabricatorEmailAddressesSettingsPanel.php index 1bdb95f568..cd1bfba540 100644 --- a/src/applications/settings/panel/PhabricatorEmailAddressesSettingsPanel.php +++ b/src/applications/settings/panel/PhabricatorEmailAddressesSettingsPanel.php @@ -138,9 +138,9 @@ final class PhabricatorEmailAddressesSettingsPanel $editable, )); - $button = null; + $buttons = array(); if ($editable) { - $button = id(new PHUIButtonView()) + $buttons[] = id(new PHUIButtonView()) ->setTag('a') ->setIcon('fa-plus') ->setText(pht('Add New Address')) @@ -149,7 +149,7 @@ final class PhabricatorEmailAddressesSettingsPanel ->setColor(PHUIButtonView::GREY); } - return $this->newBox(pht('Email Addresses'), $table, array($button)); + return $this->newBox(pht('Email Addresses'), $table, $buttons); } private function returnNewAddressResponse( diff --git a/src/docs/user/cluster/cluster_notifications.diviner b/src/docs/user/cluster/cluster_notifications.diviner index 3cdeec3c39..79c89769fc 100644 --- a/src/docs/user/cluster/cluster_notifications.diviner +++ b/src/docs/user/cluster/cluster_notifications.diviner @@ -14,7 +14,7 @@ are: - performance and capacity may improve. This configuration is relatively simple, but has a small impact on availability -and does nothing to increase resitance to data loss. +and does nothing to increase resistance to data loss. Clustering Design Goals diff --git a/src/docs/user/configuration/configuring_backups.diviner b/src/docs/user/configuration/configuring_backups.diviner index 9776448216..d32eebb0da 100644 --- a/src/docs/user/configuration/configuring_backups.diviner +++ b/src/docs/user/configuration/configuring_backups.diviner @@ -145,6 +145,24 @@ present a risk. If you restrict access to the Phabricator host or database, you should also restrict access to the backups. +Skipping Indexes +================ + +By default, `bin/storage dump` does not dump all of the data in the database: +it skips some caches which can be rebuilt automatically and do not need to be +backed up. Some of these caches are very large, so the size of the dump may +be significantly smaller than the size of the databases. + +If you have a large amount of data, you can specify `--no-indexes` when taking +a database dump to skip additional tables which contain search indexes. This +will reduce the size (and increase the speed) of the backup. This is an +advanced option which most installs will not benefit from. + +This index data can be rebuilt after a restore, but will not be rebuilt +automatically. If you choose to use this flag, you must manually rebuild +indexes after a restore (for details, see ((reindex))). + + Next Steps ========== diff --git a/src/infrastructure/graph/ManiphestTaskGraph.php b/src/infrastructure/graph/ManiphestTaskGraph.php index 24e463e273..99191760dd 100644 --- a/src/infrastructure/graph/ManiphestTaskGraph.php +++ b/src/infrastructure/graph/ManiphestTaskGraph.php @@ -27,6 +27,8 @@ final class ManiphestTaskGraph protected function newTableRow($phid, $object, $trace) { $viewer = $this->getViewer(); + Javelin::initBehavior('phui-hovercards'); + if ($object) { $status = $object->getStatus(); $priority = $object->getPriority(); @@ -51,15 +53,16 @@ final class ManiphestTaskGraph $assigned = phutil_tag('em', array(), pht('None')); } - $full_title = $object->getTitle(); - - $link = phutil_tag( + $link = javelin_tag( 'a', array( 'href' => $object->getURI(), - 'title' => $full_title, + 'sigil' => 'hovercard', + 'meta' => array( + 'hoverPHID' => $object->getPHID(), + ), ), - $full_title); + $object->getTitle()); $link = array( phutil_tag( diff --git a/src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php b/src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php index 09cb3e499c..c2b8ce04e0 100644 --- a/src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php +++ b/src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php @@ -1700,6 +1700,34 @@ abstract class PhabricatorCursorPagedPolicyAwareQuery } } + // Remove common ngrams, like "the", which occur too frequently in + // documents to be useful in constraining the query. The best ngrams + // are obscure sequences which occur in very few documents. + + if ($flat) { + $common_ngrams = queryfx_all( + $conn, + 'SELECT ngram FROM %T WHERE ngram IN (%Ls)', + $engine->getCommonNgramsTableName(), + ipull($flat, 'ngram')); + $common_ngrams = ipull($common_ngrams, 'ngram', 'ngram'); + + foreach ($flat as $key => $spec) { + $ngram = $spec['ngram']; + if (isset($common_ngrams[$ngram])) { + unset($flat[$key]); + continue; + } + + // NOTE: MySQL discards trailing whitespace in CHAR(X) columns. + $trim_ngram = rtrim($ngram, ' '); + if (isset($common_ngrams[$trim_ngram])) { + unset($flat[$key]); + continue; + } + } + } + // MySQL only allows us to join a maximum of 61 tables per query. Each // ngram is going to cost us a join toward that limit, so if the user // specified a very long query string, just pick 16 of the ngrams diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php index 4dc5c64042..fecc0517e3 100644 --- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php +++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php @@ -30,6 +30,13 @@ final class PhabricatorStorageManagementDumpWorkflow 'With __--output__, write a compressed file to disk instead '. 'of a plaintext file.'), ), + array( + 'name' => 'no-indexes', + 'help' => pht( + 'Do not dump data in rebuildable index tables. This means '. + 'backups are smaller and faster, but you will need to manually '. + 'rebuild indexes after performing a restore.'), + ), array( 'name' => 'overwrite', 'help' => pht( @@ -49,6 +56,8 @@ final class PhabricatorStorageManagementDumpWorkflow $console = PhutilConsole::getConsole(); + $with_indexes = !$args->getArg('no-indexes'); + $applied = $api->getAppliedPatches(); if ($applied === null) { $namespace = $api->getNamespace(); @@ -62,7 +71,64 @@ final class PhabricatorStorageManagementDumpWorkflow return 1; } - $databases = $api->getDatabaseList($patches, true); + $ref = $api->getRef(); + $ref_key = $ref->getRefKey(); + + $schemata_query = id(new PhabricatorConfigSchemaQuery()) + ->setAPIs(array($api)) + ->setRefs(array($ref)); + + $actual_map = $schemata_query->loadActualSchemata(); + $expect_map = $schemata_query->loadExpectedSchemata(); + + $schemata = $actual_map[$ref_key]; + $expect = $expect_map[$ref_key]; + + $targets = array(); + foreach ($schemata->getDatabases() as $database_name => $database) { + $expect_database = $expect->getDatabase($database_name); + foreach ($database->getTables() as $table_name => $table) { + + // NOTE: It's possible for us to find tables in these database which + // we don't expect to be there. For example, an older version of + // Phabricator may have had a table that was later dropped. We assume + // these are data tables and always dump them, erring on the side of + // caution. + + $persistence = PhabricatorConfigTableSchema::PERSISTENCE_DATA; + if ($expect_database) { + $expect_table = $expect_database->getTable($table_name); + if ($expect_table) { + $persistence = $expect_table->getPersistenceType(); + } + } + + switch ($persistence) { + case PhabricatorConfigTableSchema::PERSISTENCE_CACHE: + // When dumping tables, leave the data in cache tables in the + // database. This will be automatically rebuild after the data + // is restored and does not need to be persisted in backups. + $with_data = false; + break; + case PhabricatorConfigTableSchema::PERSISTENCE_INDEX: + // When dumping tables, leave index data behind of the caller + // specified "--no-indexes". These tables can be rebuilt manually + // from other tables, but do not rebuild automatically. + $with_data = $with_indexes; + break; + case PhabricatorConfigTableSchema::PERSISTENCE_DATA: + default: + $with_data = true; + break; + } + + $targets[] = array( + 'database' => $database_name, + 'table' => $table_name, + 'data' => $with_data, + ); + } + } list($host, $port) = $this->getBareHostAndPort($api->getHost()); @@ -126,35 +192,46 @@ final class PhabricatorStorageManagementDumpWorkflow $argv[] = $port; } - $argv[] = '--databases'; - foreach ($databases as $database) { - $argv[] = $database; + $commands = array(); + foreach ($targets as $target) { + $target_argv = $argv; + + if (!$target['data']) { + $target_argv[] = '--no-data'; + } + + if ($has_password) { + $commands[] = csprintf( + 'mysqldump -p%P %Ls -- %R %R', + $password, + $target_argv, + $target['database'], + $target['table']); + } else { + $command = csprintf( + 'mysqldump %Ls -- %R %R', + $target_argv, + $target['database'], + $target['table']); + } + + $commands[] = $command; } - if ($has_password) { - $command = csprintf('mysqldump -p%P %Ls', $password, $argv); - } else { - $command = csprintf('mysqldump %Ls', $argv); - } - // Decrease the CPU priority of this process so it doesn't contend with // other more important things. if (function_exists('proc_nice')) { proc_nice(19); } - - // If we aren't writing to a file, just passthru the command. - if ($output_file === null) { - return phutil_passthru('%C', $command); - } - // If we are writing to a file, stream the command output to disk. This // mode makes sure the whole command fails if there's an error (commonly, // a full disk). See T6996 for discussion. - if ($is_compress) { + if ($output_file === null) { + $file = null; + } else if ($is_compress) { $file = gzopen($output_file, 'wb1'); } else { $file = fopen($output_file, 'wb'); @@ -167,41 +244,47 @@ final class PhabricatorStorageManagementDumpWorkflow $file)); } - $future = new ExecFuture('%C', $command); - try { - $iterator = id(new FutureIterator(array($future))) - ->setUpdateInterval(0.100); - foreach ($iterator as $ready) { - list($stdout, $stderr) = $future->read(); - $future->discardBuffers(); + foreach ($commands as $command) { + $future = new ExecFuture('%C', $command); - if (strlen($stderr)) { - fwrite(STDERR, $stderr); - } + $iterator = id(new FutureIterator(array($future))) + ->setUpdateInterval(0.100); + foreach ($iterator as $ready) { + list($stdout, $stderr) = $future->read(); + $future->discardBuffers(); - if (strlen($stdout)) { - if ($is_compress) { - $ok = gzwrite($file, $stdout); - } else { - $ok = fwrite($file, $stdout); + if (strlen($stderr)) { + fwrite(STDERR, $stderr); } - if ($ok !== strlen($stdout)) { - throw new Exception( - pht( - 'Failed to write %d byte(s) to file "%s".', - new PhutilNumber(strlen($stdout)), - $output_file)); - } - } + if (strlen($stdout)) { + if (!$file) { + $ok = fwrite(STDOUT, $stdout); + } else if ($is_compress) { + $ok = gzwrite($file, $stdout); + } else { + $ok = fwrite($file, $stdout); + } - if ($ready !== null) { - $ready->resolvex(); + if ($ok !== strlen($stdout)) { + throw new Exception( + pht( + 'Failed to write %d byte(s) to file "%s".', + new PhutilNumber(strlen($stdout)), + $output_file)); + } + } + + if ($ready !== null) { + $ready->resolvex(); + } } } - if ($is_compress) { + if (!$file) { + $ok = true; + } else if ($is_compress) { $ok = gzclose($file); } else { $ok = fclose($file); @@ -218,7 +301,9 @@ final class PhabricatorStorageManagementDumpWorkflow // we don't leave any confusing artifacts laying around. try { - Filesystem::remove($output_file); + if ($file !== null) { + Filesystem::remove($output_file); + } } catch (Exception $ex) { // Ignore any errors we hit. }