revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-12-15 10:00:55 +01:00
Code Issues Releases Wiki Activity

(stable) Promote 2017 Week 40

Browse source
This commit is contained in:
epriestley 2017-10-06 14:19:27 -07:00
commit 58598e1963
36 changed files with 690 additions and 81 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
resources/sql/autopatches/20171002.cngram.01.maniphest.sql Normal file
View file

@ -0,0 +1,7 @@
CREATE TABLE {$NAMESPACE}_maniphest.maniphest_task_fngrams_common (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
needsCollection BOOL NOT NULL,
UNIQUE KEY `key_ngram` (ngram),
KEY `key_collect` (needsCollection)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

7
resources/sql/autopatches/20171002.cngram.02.event.sql Normal file
View file

@ -0,0 +1,7 @@
CREATE TABLE {$NAMESPACE}_calendar.calendar_event_fngrams_common (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
needsCollection BOOL NOT NULL,
UNIQUE KEY `key_ngram` (ngram),
KEY `key_collect` (needsCollection)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

7
resources/sql/autopatches/20171002.cngram.03.revision.sql Normal file
View file

@ -0,0 +1,7 @@
CREATE TABLE {$NAMESPACE}_differential.differential_revision_fngrams_common (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
needsCollection BOOL NOT NULL,
UNIQUE KEY `key_ngram` (ngram),
KEY `key_collect` (needsCollection)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

7
resources/sql/autopatches/20171002.cngram.04.fund.sql Normal file
View file

@ -0,0 +1,7 @@
CREATE TABLE {$NAMESPACE}_fund.fund_initiative_fngrams_common (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
needsCollection BOOL NOT NULL,
UNIQUE KEY `key_ngram` (ngram),
KEY `key_collect` (needsCollection)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

7
resources/sql/autopatches/20171002.cngram.05.owners.sql Normal file
View file

@ -0,0 +1,7 @@
CREATE TABLE {$NAMESPACE}_owners.owners_package_fngrams_common (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
needsCollection BOOL NOT NULL,
UNIQUE KEY `key_ngram` (ngram),
KEY `key_collect` (needsCollection)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

7
resources/sql/autopatches/20171002.cngram.06.passphrase.sql Normal file
View file

@ -0,0 +1,7 @@
CREATE TABLE {$NAMESPACE}_passphrase.passphrase_credential_fngrams_common (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
needsCollection BOOL NOT NULL,
UNIQUE KEY `key_ngram` (ngram),
KEY `key_collect` (needsCollection)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

7
resources/sql/autopatches/20171002.cngram.07.blog.sql Normal file
View file

@ -0,0 +1,7 @@
CREATE TABLE {$NAMESPACE}_phame.phame_blog_fngrams_common (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
needsCollection BOOL NOT NULL,
UNIQUE KEY `key_ngram` (ngram),
KEY `key_collect` (needsCollection)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

7
resources/sql/autopatches/20171002.cngram.08.post.sql Normal file
View file

@ -0,0 +1,7 @@
CREATE TABLE {$NAMESPACE}_phame.phame_post_fngrams_common (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
needsCollection BOOL NOT NULL,
UNIQUE KEY `key_ngram` (ngram),
KEY `key_collect` (needsCollection)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

7
resources/sql/autopatches/20171002.cngram.09.pholio.sql Normal file
View file

@ -0,0 +1,7 @@
CREATE TABLE {$NAMESPACE}_pholio.pholio_mock_fngrams_common (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
needsCollection BOOL NOT NULL,
UNIQUE KEY `key_ngram` (ngram),
KEY `key_collect` (needsCollection)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

7
resources/sql/autopatches/20171002.cngram.10.phriction.sql Normal file
View file

@ -0,0 +1,7 @@
CREATE TABLE {$NAMESPACE}_phriction.phriction_document_fngrams_common (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
needsCollection BOOL NOT NULL,
UNIQUE KEY `key_ngram` (ngram),
KEY `key_collect` (needsCollection)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

7
resources/sql/autopatches/20171002.cngram.11.project.sql Normal file
View file

@ -0,0 +1,7 @@
CREATE TABLE {$NAMESPACE}_project.project_project_fngrams_common (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
needsCollection BOOL NOT NULL,
UNIQUE KEY `key_ngram` (ngram),
KEY `key_collect` (needsCollection)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

7
resources/sql/autopatches/20171002.cngram.12.user.sql Normal file
View file

@ -0,0 +1,7 @@
CREATE TABLE {$NAMESPACE}_user.user_user_fngrams_common (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
needsCollection BOOL NOT NULL,
UNIQUE KEY `key_ngram` (ngram),
KEY `key_collect` (needsCollection)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

7
resources/sql/autopatches/20171002.cngram.13.repository.sql Normal file
View file

@ -0,0 +1,7 @@
CREATE TABLE {$NAMESPACE}_repository.repository_repository_fngrams_common (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
needsCollection BOOL NOT NULL,
UNIQUE KEY `key_ngram` (ngram),
KEY `key_collect` (needsCollection)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

7
resources/sql/autopatches/20171002.cngram.14.commit.sql Normal file
View file

@ -0,0 +1,7 @@
CREATE TABLE {$NAMESPACE}_repository.repository_commit_fngrams_common (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT},
needsCollection BOOL NOT NULL,
UNIQUE KEY `key_ngram` (ngram),
KEY `key_collect` (needsCollection)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

6
src/__phutil_library_map__.php
View file

@ -3941,6 +3941,7 @@ phutil_register_library_map(array(
'PhabricatorSearchEngineAttachment' => 'applications/search/engineextension/PhabricatorSearchEngineAttachment.php',
'PhabricatorSearchEngineExtension' => 'applications/search/engineextension/PhabricatorSearchEngineExtension.php',
'PhabricatorSearchEngineExtensionModule' => 'applications/search/engineextension/PhabricatorSearchEngineExtensionModule.php',
'PhabricatorSearchFerretNgramGarbageCollector' => 'applications/search/garbagecollector/PhabricatorSearchFerretNgramGarbageCollector.php',
'PhabricatorSearchField' => 'applications/search/field/PhabricatorSearchField.php',
'PhabricatorSearchHost' => 'infrastructure/cluster/search/PhabricatorSearchHost.php',
'PhabricatorSearchHovercardController' => 'applications/search/controller/PhabricatorSearchHovercardController.php',
@ -3948,6 +3949,8 @@ phutil_register_library_map(array(
'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'applications/search/engineextension/PhabricatorSearchIndexVersionDestructionEngineExtension.php',
'PhabricatorSearchManagementIndexWorkflow' => 'applications/search/management/PhabricatorSearchManagementIndexWorkflow.php',
'PhabricatorSearchManagementInitWorkflow' => 'applications/search/management/PhabricatorSearchManagementInitWorkflow.php',
'PhabricatorSearchManagementNgramsWorkflow' => 'applications/search/management/PhabricatorSearchManagementNgramsWorkflow.php',
'PhabricatorSearchManagementQueryWorkflow' => 'applications/search/management/PhabricatorSearchManagementQueryWorkflow.php',
'PhabricatorSearchManagementWorkflow' => 'applications/search/management/PhabricatorSearchManagementWorkflow.php',
'PhabricatorSearchNgrams' => 'applications/search/ngrams/PhabricatorSearchNgrams.php',
'PhabricatorSearchNgramsDestructionEngineExtension' => 'applications/search/engineextension/PhabricatorSearchNgramsDestructionEngineExtension.php',
@ -9521,6 +9524,7 @@ phutil_register_library_map(array(
'PhabricatorSearchEngineAttachment' => 'Phobject',
'PhabricatorSearchEngineExtension' => 'Phobject',
'PhabricatorSearchEngineExtensionModule' => 'PhabricatorConfigModule',
'PhabricatorSearchFerretNgramGarbageCollector' => 'PhabricatorGarbageCollector',
'PhabricatorSearchField' => 'Phobject',
'PhabricatorSearchHost' => 'Phobject',
'PhabricatorSearchHovercardController' => 'PhabricatorSearchBaseController',
@ -9528,6 +9532,8 @@ phutil_register_library_map(array(
'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'PhabricatorDestructionEngineExtension',
'PhabricatorSearchManagementIndexWorkflow' => 'PhabricatorSearchManagementWorkflow',
'PhabricatorSearchManagementInitWorkflow' => 'PhabricatorSearchManagementWorkflow',
'PhabricatorSearchManagementNgramsWorkflow' => 'PhabricatorSearchManagementWorkflow',
'PhabricatorSearchManagementQueryWorkflow' => 'PhabricatorSearchManagementWorkflow',
'PhabricatorSearchManagementWorkflow' => 'PhabricatorManagementWorkflow',
'PhabricatorSearchNgrams' => 'PhabricatorSearchDAO',
'PhabricatorSearchNgramsDestructionEngineExtension' => 'PhabricatorDestructionEngineExtension',

3
src/applications/cache/storage/PhabricatorCacheSchemaSpec.php vendored
View file

@ -30,6 +30,9 @@ final class PhabricatorCacheSchemaSpec extends PhabricatorConfigSchemaSpec {
'key_ttl' => array(
'columns' => array('cacheExpires'),
),
),
array(
'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_CACHE,
));
}

3
src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php
View file

@ -261,6 +261,7 @@ final class PhabricatorConfigDatabaseStatusController
$this->renderAttr(
$table->getCollation(),
$table->hasIssue($collation_issue)),
$table->getPersistenceTypeDisplayName(),
);
}
@ -270,12 +271,14 @@ final class PhabricatorConfigDatabaseStatusController
null,
pht('Table'),
pht('Collation'),
pht('Persistence'),
))
->setColumnClasses(
array(
null,
'wide pri',
null,
null,
));
$title = $database_name;

2
src/applications/config/schema/PhabricatorConfigSchemaQuery.php
View file

@ -338,6 +338,8 @@ final class PhabricatorConfigSchemaQuery extends Phobject {
$comp_table->addKey($comp_key);
}
$comp_table->setPersistenceType($expect_table->getPersistenceType());
$comp_database->addTable($comp_table);
}
$comp_server->addDatabase($comp_database);

35
src/applications/config/schema/PhabricatorConfigSchemaSpec.php
View file

@ -56,30 +56,52 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject {
}
protected function buildFerretIndexSchema(PhabricatorFerretEngine $engine) {
$index_options = array(
'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_INDEX,
);
$this->buildRawSchema(
$engine->getApplicationName(),
$engine->getDocumentTableName(),
$engine->getDocumentSchemaColumns(),
$engine->getDocumentSchemaKeys());
$engine->getDocumentSchemaKeys(),
$index_options);
$this->buildRawSchema(
$engine->getApplicationName(),
$engine->getFieldTableName(),
$engine->getFieldSchemaColumns(),
$engine->getFieldSchemaKeys());
$engine->getFieldSchemaKeys(),
$index_options);
$this->buildRawSchema(
$engine->getApplicationName(),
$engine->getNgramsTableName(),
$engine->getNgramsSchemaColumns(),
$engine->getNgramsSchemaKeys());
$engine->getNgramsSchemaKeys(),
$index_options);
$this->buildRawSchema(
$engine->getApplicationName(),
$engine->getCommonNgramsTableName(),
$engine->getCommonNgramsSchemaColumns(),
$engine->getCommonNgramsSchemaKeys(),
$index_options);
}
protected function buildRawSchema(
$database_name,
$table_name,
array $columns,
array $keys) {
array $keys,
array $options = array()) {
PhutilTypeSpec::checkMap(
$options,
array(
'persistence' => 'optional string',
));
$database = $this->getDatabase($database_name);
$table = $this->newTable($table_name);
@ -138,6 +160,11 @@ abstract class PhabricatorConfigSchemaSpec extends Phobject {
$table->addKey($key);
}
$persistence_type = idx($options, 'persistence');
if ($persistence_type !== null) {
$table->setPersistenceType($persistence_type);
}
$database->addTable($table);
}

26
src/applications/config/schema/PhabricatorConfigTableSchema.php
View file

@ -7,6 +7,11 @@ final class PhabricatorConfigTableSchema
private $engine;
private $columns = array();
private $keys = array();
private $persistenceType = self::PERSISTENCE_DATA;
const PERSISTENCE_DATA = 'data';
const PERSISTENCE_CACHE = 'cache';
const PERSISTENCE_INDEX = 'index';
public function addColumn(PhabricatorConfigColumnSchema $column) {
$key = $column->getName();
@ -45,6 +50,27 @@ final class PhabricatorConfigTableSchema
return idx($this->getKeys(), $key);
}
public function setPersistenceType($persistence_type) {
$this->persistenceType = $persistence_type;
return $this;
}
public function getPersistenceType() {
return $this->persistenceType;
}
public function getPersistenceTypeDisplayName() {
$map = array(
self::PERSISTENCE_DATA => pht('Data'),
self::PERSISTENCE_CACHE => pht('Cache'),
self::PERSISTENCE_INDEX => pht('Index'),
);
$type = $this->getPersistenceType();
return idx($map, $type, $type);
}
protected function getSubschemata() {
// NOTE: Keys and columns may have the same name, so make sure we return
// everything.

3
src/applications/differential/storage/DifferentialSchemaSpec.php
View file

@ -21,6 +21,9 @@ final class DifferentialSchemaSpec extends PhabricatorConfigSchemaSpec {
'dateCreated' => array(
'columns' => array('dateCreated'),
),
),
array(
'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_CACHE,
));
$this->buildRawSchema(

9
src/applications/diffusion/application/PhabricatorDiffusionApplication.php
View file

@ -69,10 +69,11 @@ final class PhabricatorDiffusionApplication extends PhabricatorApplication {
'branches/(?P<dblob>.*)' => 'DiffusionBranchTableController',
'refs/(?P<dblob>.*)' => 'DiffusionRefTableController',
'lint/(?P<dblob>.*)' => 'DiffusionLintController',
'commit/(?P<commit>[a-z0-9]+)/branches/'
=> 'DiffusionCommitBranchesController',
'commit/(?P<commit>[a-z0-9]+)/tags/'
=> 'DiffusionCommitTagsController',
'commit/(?P<commit>[a-z0-9]+)' => array(
'/?' => 'DiffusionCommitController',
'/branches/' => 'DiffusionCommitBranchesController',
'/tags/' => 'DiffusionCommitTagsController',
),
'compare/' => 'DiffusionCompareController',
'manage/(?:(?P<panel>[^/]+)/)?'
=> 'DiffusionRepositoryManagePanelsController',

14
src/applications/diffusion/conduit/DiffusionLastModifiedQueryConduitAPIMethod.php
View file

@ -115,6 +115,10 @@ final class DiffusionLastModifiedQueryConduitAPIMethod
$graph_cache = new PhabricatorRepositoryGraphCache();
$results = array();
// Spend no more than this many total seconds trying to satisfy queries
// via the graph cache.
$remaining_time = 10.0;
foreach ($map as $path => $commit) {
$path_id = idx($path_map, $path);
if (!$path_id) {
@ -125,13 +129,21 @@ final class DiffusionLastModifiedQueryConduitAPIMethod
continue;
}
$t_start = microtime(true);
$cache_result = $graph_cache->loadLastModifiedCommitID(
$commit_id,
$path_id);
$path_id,
$remaining_time);
$t_end = microtime(true);
if ($cache_result !== false) {
$results[$path] = $cache_result;
}
$remaining_time -= ($t_end - $t_start);
if ($remaining_time <= 0) {
break;
}
}
if ($results) {

16
src/applications/diffusion/controller/DiffusionCommitController.php
View file

@ -22,17 +22,27 @@ final class DiffusionCommitController extends DiffusionController {
$drequest = $this->getDiffusionRequest();
$viewer = $request->getUser();
$repository = $drequest->getRepository();
$commit_identifier = $drequest->getCommit();
// If this page is being accessed via "/source/xyz/commit/...", redirect
// to the canonical URI.
$has_callsign = strlen($request->getURIData('repositoryCallsign'));
$has_id = strlen($request->getURIData('repositoryID'));
if (!$has_callsign && !$has_id) {
$canonical_uri = $repository->getCommitURI($commit_identifier);
return id(new AphrontRedirectResponse())
->setURI($canonical_uri);
}
if ($request->getStr('diff')) {
return $this->buildRawDiffResponse($drequest);
}
$repository = $drequest->getRepository();
$commit = id(new DiffusionCommitQuery())
->setViewer($viewer)
->withRepository($repository)
->withIdentifiers(array($drequest->getCommit()))
->withIdentifiers(array($commit_identifier))
->needCommitData(true)
->needAuditRequests(true)
->executeOne();

17
src/applications/repository/graphcache/PhabricatorRepositoryGraphCache.php
View file

@ -102,6 +102,10 @@ final class PhabricatorRepositoryGraphCache extends Phobject {
}
// Otherwise, the rebuild gave us the data, so we can keep going.
$did_fill = true;
} else {
$did_fill = false;
}
// Sanity check so we can survive and recover from bad data.
@ -147,12 +151,17 @@ final class PhabricatorRepositoryGraphCache extends Phobject {
$commit_id = $parent_id;
// Periodically check if we've spent too long looking for a result
// in the cache, and return so we can fall back to a VCS operation. This
// keeps us from having a degenerate worst case if, e.g., the cache
// is cold and we need to inspect a very large number of blocks
// in the cache, and return so we can fall back to a VCS operation.
// This keeps us from having a degenerate worst case if, e.g., the
// cache is cold and we need to inspect a very large number of blocks
// to satisfy the query.
if (((++$iterations) % 64) === 0) {
++$iterations;
// If we performed a cache fill in this cycle, always check the time
// limit, since cache fills may take a significant amount of time.
if ($did_fill || ($iterations % 64 === 0)) {
$t_end = microtime(true);
if (($t_end - $t_start) > $time) {
return false;

49
src/applications/search/engineextension/PhabricatorFerretFulltextEngineExtension.php
View file

@ -165,21 +165,46 @@ final class PhabricatorFerretFulltextEngineExtension
$ferret_field['normalCorpus']);
}
$sql = array();
foreach ($ngrams as $ngram) {
$sql[] = qsprintf(
if ($ngrams) {
$common = queryfx_all(
$conn,
'(%d, %s)',
$document_id,
$ngram);
'SELECT ngram FROM %T WHERE ngram IN (%Ls)',
$engine->getCommonNgramsTableName(),
$ngrams);
$common = ipull($common, 'ngram', 'ngram');
foreach ($ngrams as $key => $ngram) {
if (isset($common[$ngram])) {
unset($ngrams[$key]);
continue;
}
// NOTE: MySQL discards trailing whitespace in CHAR(X) columns.
$trim_ngram = rtrim($ngram, ' ');
if (isset($common[$ngram])) {
unset($ngrams[$key]);
continue;
}
}
}
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
queryfx(
$conn,
'INSERT INTO %T (documentID, ngram) VALUES %Q',
$engine->getNgramsTableName(),
$chunk);
if ($ngrams) {
$sql = array();
foreach ($ngrams as $ngram) {
$sql[] = qsprintf(
$conn,
'(%d, %s)',
$document_id,
$ngram);
}
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
queryfx(
$conn,
'INSERT INTO %T (documentID, ngram) VALUES %Q',
$engine->getNgramsTableName(),
$chunk);
}
}
} catch (Exception $ex) {
$object->killTransaction();

31
src/applications/search/ferret/PhabricatorFerretEngine.php
View file

@ -295,4 +295,35 @@ abstract class PhabricatorFerretEngine extends Phobject {
);
}
public function getCommonNgramsTableName() {
$application = $this->getApplicationName();
$scope = $this->getScopeName();
return "{$application}_{$scope}_fngrams_common";
}
public function getCommonNgramsSchemaColumns() {
return array(
'id' => 'auto',
'ngram' => 'char3',
'needsCollection' => 'bool',
);
}
public function getCommonNgramsSchemaKeys() {
return array(
'PRIMARY' => array(
'columns' => array('id'),
'unique' => true,
),
'key_ngram' => array(
'columns' => array('ngram'),
'unique' => true,
),
'key_collect' => array(
'columns' => array('needsCollection'),
),
);
}
}

55
src/applications/search/garbagecollector/PhabricatorSearchFerretNgramGarbageCollector.php Normal file
View file

@ -0,0 +1,55 @@
<?php
final class PhabricatorSearchFerretNgramGarbageCollector
extends PhabricatorGarbageCollector {
const COLLECTORCONST = 'search.ferret.ngram';
public function getCollectorName() {
return pht('Ferret Engine Ngrams');
}
public function hasAutomaticPolicy() {
return true;
}
protected function collectGarbage() {
$all_objects = id(new PhutilClassMapQuery())
->setAncestorClass('PhabricatorFerretInterface')
->execute();
$did_collect = false;
foreach ($all_objects as $object) {
$engine = $object->newFerretEngine();
$conn = $object->establishConnection('w');
$ngram_row = queryfx_one(
$conn,
'SELECT ngram FROM %T WHERE needsCollection = 1 LIMIT 1',
$engine->getCommonNgramsTableName());
if (!$ngram_row) {
continue;
}
$ngram = $ngram_row['ngram'];
queryfx(
$conn,
'DELETE FROM %T WHERE ngram = %s',
$engine->getNgramsTableName(),
$ngram);
queryfx(
$conn,
'UPDATE %T SET needsCollection = 0 WHERE ngram = %s',
$engine->getCommonNgramsTableName(),
$ngram);
$did_collect = true;
break;
}
return $did_collect;
}
}

109
src/applications/search/management/PhabricatorSearchManagementNgramsWorkflow.php Normal file
View file

@ -0,0 +1,109 @@
<?php
final class PhabricatorSearchManagementNgramsWorkflow
extends PhabricatorSearchManagementWorkflow {
protected function didConstruct() {
$this
->setName('ngrams')
->setSynopsis(
pht(
'Recompute common ngrams. This is an advanced workflow that '.
'can harm search quality if used improperly.'))
->setArguments(
array(
array(
'name' => 'reset',
'help' => pht('Reset all common ngram records.'),
),
));
}
public function execute(PhutilArgumentParser $args) {
$is_reset = $args->getArg('reset');
$all_objects = id(new PhutilClassMapQuery())
->setAncestorClass('PhabricatorFerretInterface')
->execute();
$min_documents = 4096;
$threshold = 0.15;
foreach ($all_objects as $object) {
$engine = $object->newFerretEngine();
$conn = $object->establishConnection('w');
$display_name = get_class($object);
if ($is_reset) {
echo tsprintf(
"%s\n",
pht(
'Resetting common ngrams for "%s".',
$display_name));
queryfx(
$conn,
'DELETE FROM %T',
$engine->getCommonNgramsTableName());
continue;
}
$document_count = queryfx_one(
$conn,
'SELECT COUNT(*) N FROM %T',
$engine->getDocumentTableName());
$document_count = $document_count['N'];
if ($document_count < $min_documents) {
echo tsprintf(
"%s\n",
pht(
'Too few documents of type "%s" for any ngrams to be common.',
$display_name));
continue;
}
$min_frequency = (int)ceil($document_count * $threshold);
$common_ngrams = queryfx_all(
$conn,
'SELECT ngram, COUNT(*) N FROM %T
GROUP BY ngram
HAVING N >= %d',
$engine->getNgramsTableName(),
$min_frequency);
if (!$common_ngrams) {
echo tsprintf(
"%s\n",
pht(
'No new common ngrams exist for "%s".',
$display_name));
continue;
}
$sql = array();
foreach ($common_ngrams as $ngram) {
$sql[] = qsprintf(
$conn,
'(%s, 1)',
$ngram['ngram']);
}
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
queryfx(
$conn,
'INSERT IGNORE INTO %T (ngram, needsCollection)
VALUES %Q',
$engine->getCommonNgramsTableName(),
$chunk);
}
echo tsprintf(
"%s\n",
pht(
'Updated common ngrams for "%s".',
$display_name));
}
}
}

55
src/applications/search/management/PhabricatorSearchManagementQueryWorkflow.php Normal file
View file

@ -0,0 +1,55 @@
<?php
final class PhabricatorSearchManagementQueryWorkflow
extends PhabricatorSearchManagementWorkflow {
protected function didConstruct() {
$this
->setName('query')
->setSynopsis(
pht('Run a search query. Intended for debugging and development.'))
->setArguments(
array(
array(
'name' => 'query',
'param' => 'query',
'help' => pht('Raw query to execute.'),
),
));
}
public function execute(PhutilArgumentParser $args) {
$viewer = $this->getViewer();
$raw_query = $args->getArg('query');
if (!strlen($raw_query)) {
throw new PhutilArgumentUsageException(
pht('Specify a query with --query.'));
}
$engine = id(new PhabricatorSearchApplicationSearchEngine())
->setViewer($viewer);
$saved = $engine->newSavedQuery();
$saved->setParameter('query', $raw_query);
$query = $engine->buildQueryFromSavedQuery($saved);
$pager = $engine->newPagerForSavedQuery($saved);
$results = $engine->executeQuery($query, $pager);
if ($results) {
foreach ($results as $result) {
echo tsprintf(
"%s\t%s\n",
$result->getPHID(),
$result->getName());
}
} else {
echo tsprintf(
"%s\n",
pht('No results.'));
}
return 0;
}
}

6
src/applications/settings/panel/PhabricatorEmailAddressesSettingsPanel.php
View file

@ -138,9 +138,9 @@ final class PhabricatorEmailAddressesSettingsPanel
$editable,
));
$button = null;
$buttons = array();
if ($editable) {
$button = id(new PHUIButtonView())
$buttons[] = id(new PHUIButtonView())
->setTag('a')
->setIcon('fa-plus')
->setText(pht('Add New Address'))
@ -149,7 +149,7 @@ final class PhabricatorEmailAddressesSettingsPanel
->setColor(PHUIButtonView::GREY);
}
return $this->newBox(pht('Email Addresses'), $table, array($button));
return $this->newBox(pht('Email Addresses'), $table, $buttons);
}
private function returnNewAddressResponse(

2
src/docs/user/cluster/cluster_notifications.diviner
View file

@ -14,7 +14,7 @@ are:
- performance and capacity may improve.
This configuration is relatively simple, but has a small impact on availability
and does nothing to increase resitance to data loss.
and does nothing to increase resistance to data loss.
Clustering Design Goals

18
src/docs/user/configuration/configuring_backups.diviner
View file

@ -145,6 +145,24 @@ present a risk. If you restrict access to the Phabricator host or database, you
should also restrict access to the backups.
Skipping Indexes
================
By default, `bin/storage dump` does not dump all of the data in the database:
it skips some caches which can be rebuilt automatically and do not need to be
backed up. Some of these caches are very large, so the size of the dump may
be significantly smaller than the size of the databases.
If you have a large amount of data, you can specify `--no-indexes` when taking
a database dump to skip additional tables which contain search indexes. This
will reduce the size (and increase the speed) of the backup. This is an
advanced option which most installs will not benefit from.
This index data can be rebuilt after a restore, but will not be rebuilt
automatically. If you choose to use this flag, you must manually rebuild
indexes after a restore (for details, see ((reindex))).
Next Steps
==========

13
src/infrastructure/graph/ManiphestTaskGraph.php
View file

@ -27,6 +27,8 @@ final class ManiphestTaskGraph
protected function newTableRow($phid, $object, $trace) {
$viewer = $this->getViewer();
Javelin::initBehavior('phui-hovercards');
if ($object) {
$status = $object->getStatus();
$priority = $object->getPriority();
@ -51,15 +53,16 @@ final class ManiphestTaskGraph
$assigned = phutil_tag('em', array(), pht('None'));
}
$full_title = $object->getTitle();
$link = phutil_tag(
$link = javelin_tag(
'a',
array(
'href' => $object->getURI(),
'title' => $full_title,
'sigil' => 'hovercard',
'meta' => array(
'hoverPHID' => $object->getPHID(),
),
),
$full_title);
$object->getTitle());
$link = array(
phutil_tag(

28
src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php
View file

@ -1700,6 +1700,34 @@ abstract class PhabricatorCursorPagedPolicyAwareQuery
}
}
// Remove common ngrams, like "the", which occur too frequently in
// documents to be useful in constraining the query. The best ngrams
// are obscure sequences which occur in very few documents.
if ($flat) {
$common_ngrams = queryfx_all(
$conn,
'SELECT ngram FROM %T WHERE ngram IN (%Ls)',
$engine->getCommonNgramsTableName(),
ipull($flat, 'ngram'));
$common_ngrams = ipull($common_ngrams, 'ngram', 'ngram');
foreach ($flat as $key => $spec) {
$ngram = $spec['ngram'];
if (isset($common_ngrams[$ngram])) {
unset($flat[$key]);
continue;
}
// NOTE: MySQL discards trailing whitespace in CHAR(X) columns.
$trim_ngram = rtrim($ngram, ' ');
if (isset($common_ngrams[$trim_ngram])) {
unset($flat[$key]);
continue;
}
}
}
// MySQL only allows us to join a maximum of 61 tables per query. Each
// ngram is going to cost us a join toward that limit, so if the user
// specified a very long query string, just pick 16 of the ngrams

173
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php
View file

@ -30,6 +30,13 @@ final class PhabricatorStorageManagementDumpWorkflow
'With __--output__, write a compressed file to disk instead '.
'of a plaintext file.'),
),
array(
'name' => 'no-indexes',
'help' => pht(
'Do not dump data in rebuildable index tables. This means '.
'backups are smaller and faster, but you will need to manually '.
'rebuild indexes after performing a restore.'),
),
array(
'name' => 'overwrite',
'help' => pht(
@ -49,6 +56,8 @@ final class PhabricatorStorageManagementDumpWorkflow
$console = PhutilConsole::getConsole();
$with_indexes = !$args->getArg('no-indexes');
$applied = $api->getAppliedPatches();
if ($applied === null) {
$namespace = $api->getNamespace();
@ -62,7 +71,64 @@ final class PhabricatorStorageManagementDumpWorkflow
return 1;
}
$databases = $api->getDatabaseList($patches, true);
$ref = $api->getRef();
$ref_key = $ref->getRefKey();
$schemata_query = id(new PhabricatorConfigSchemaQuery())
->setAPIs(array($api))
->setRefs(array($ref));
$actual_map = $schemata_query->loadActualSchemata();
$expect_map = $schemata_query->loadExpectedSchemata();
$schemata = $actual_map[$ref_key];
$expect = $expect_map[$ref_key];
$targets = array();
foreach ($schemata->getDatabases() as $database_name => $database) {
$expect_database = $expect->getDatabase($database_name);
foreach ($database->getTables() as $table_name => $table) {
// NOTE: It's possible for us to find tables in these database which
// we don't expect to be there. For example, an older version of
// Phabricator may have had a table that was later dropped. We assume
// these are data tables and always dump them, erring on the side of
// caution.
$persistence = PhabricatorConfigTableSchema::PERSISTENCE_DATA;
if ($expect_database) {
$expect_table = $expect_database->getTable($table_name);
if ($expect_table) {
$persistence = $expect_table->getPersistenceType();
}
}
switch ($persistence) {
case PhabricatorConfigTableSchema::PERSISTENCE_CACHE:
// When dumping tables, leave the data in cache tables in the
// database. This will be automatically rebuild after the data
// is restored and does not need to be persisted in backups.
$with_data = false;
break;
case PhabricatorConfigTableSchema::PERSISTENCE_INDEX:
// When dumping tables, leave index data behind of the caller
// specified "--no-indexes". These tables can be rebuilt manually
// from other tables, but do not rebuild automatically.
$with_data = $with_indexes;
break;
case PhabricatorConfigTableSchema::PERSISTENCE_DATA:
default:
$with_data = true;
break;
}
$targets[] = array(
'database' => $database_name,
'table' => $table_name,
'data' => $with_data,
);
}
}
list($host, $port) = $this->getBareHostAndPort($api->getHost());
@ -126,35 +192,46 @@ final class PhabricatorStorageManagementDumpWorkflow
$argv[] = $port;
}
$argv[] = '--databases';
foreach ($databases as $database) {
$argv[] = $database;
$commands = array();
foreach ($targets as $target) {
$target_argv = $argv;
if (!$target['data']) {
$target_argv[] = '--no-data';
}
if ($has_password) {
$commands[] = csprintf(
'mysqldump -p%P %Ls -- %R %R',
$password,
$target_argv,
$target['database'],
$target['table']);
} else {
$command = csprintf(
'mysqldump %Ls -- %R %R',
$target_argv,
$target['database'],
$target['table']);
}
$commands[] = $command;
}
if ($has_password) {
$command = csprintf('mysqldump -p%P %Ls', $password, $argv);
} else {
$command = csprintf('mysqldump %Ls', $argv);
}
// Decrease the CPU priority of this process so it doesn't contend with
// other more important things.
if (function_exists('proc_nice')) {
proc_nice(19);
}
// If we aren't writing to a file, just passthru the command.
if ($output_file === null) {
return phutil_passthru('%C', $command);
}
// If we are writing to a file, stream the command output to disk. This
// mode makes sure the whole command fails if there's an error (commonly,
// a full disk). See T6996 for discussion.
if ($is_compress) {
if ($output_file === null) {
$file = null;
} else if ($is_compress) {
$file = gzopen($output_file, 'wb1');
} else {
$file = fopen($output_file, 'wb');
@ -167,41 +244,47 @@ final class PhabricatorStorageManagementDumpWorkflow
$file));
}
$future = new ExecFuture('%C', $command);
try {
$iterator = id(new FutureIterator(array($future)))
->setUpdateInterval(0.100);
foreach ($iterator as $ready) {
list($stdout, $stderr) = $future->read();
$future->discardBuffers();
foreach ($commands as $command) {
$future = new ExecFuture('%C', $command);
if (strlen($stderr)) {
fwrite(STDERR, $stderr);
}
$iterator = id(new FutureIterator(array($future)))
->setUpdateInterval(0.100);
foreach ($iterator as $ready) {
list($stdout, $stderr) = $future->read();
$future->discardBuffers();
if (strlen($stdout)) {
if ($is_compress) {
$ok = gzwrite($file, $stdout);
} else {
$ok = fwrite($file, $stdout);
if (strlen($stderr)) {
fwrite(STDERR, $stderr);
}
if ($ok !== strlen($stdout)) {
throw new Exception(
pht(
'Failed to write %d byte(s) to file "%s".',
new PhutilNumber(strlen($stdout)),
$output_file));
}
}
if (strlen($stdout)) {
if (!$file) {
$ok = fwrite(STDOUT, $stdout);
} else if ($is_compress) {
$ok = gzwrite($file, $stdout);
} else {
$ok = fwrite($file, $stdout);
}
if ($ready !== null) {
$ready->resolvex();
if ($ok !== strlen($stdout)) {
throw new Exception(
pht(
'Failed to write %d byte(s) to file "%s".',
new PhutilNumber(strlen($stdout)),
$output_file));
}
}
if ($ready !== null) {
$ready->resolvex();
}
}
}
if ($is_compress) {
if (!$file) {
$ok = true;
} else if ($is_compress) {
$ok = gzclose($file);
} else {
$ok = fclose($file);
@ -218,7 +301,9 @@ final class PhabricatorStorageManagementDumpWorkflow
// we don't leave any confusing artifacts laying around.
try {
Filesystem::remove($output_file);
if ($file !== null) {
Filesystem::remove($output_file);
}
} catch (Exception $ex) {
// Ignore any errors we hit.
}