From 2a3c3b2b98fe922969ebad5ac6521397259cd655 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 8 Mar 2016 06:27:26 -0800 Subject: [PATCH] Provide `bin/nuance import` and ngram indexes for sources Summary: Ref T10537. More infrastructure: - Put a `bin/nuance` in place with `bin/nuance import`. This has no useful behavior yet. - Allow sources to be searched by substring. This supports `bin/nuance import --source whatever` so you don't have to dig up PHIDs. Test Plan: - Applied migrations. - Ran `bin/nuance import --source ...` (no meaningful effect, but works fine). - Searched for sources by substring in the UI. Reviewers: chad Reviewed By: chad Maniphest Tasks: T10537 Differential Revision: https://secure.phabricator.com/D15436 --- bin/nuance | 1 + .../20160308.nuance.03.sourcen.sql | 7 ++ .../20160308.nuance.04.sourcei.php | 11 +++ .../20160308.nuance.05.sourcename.sql | 2 + scripts/setup/manage_nuance.php | 21 ++++++ src/__phutil_library_map__.php | 7 ++ .../almanac/storage/AlmanacDevice.php | 2 +- .../almanac/storage/AlmanacNamespace.php | 2 +- .../almanac/storage/AlmanacNetwork.php | 2 +- .../almanac/storage/AlmanacService.php | 2 +- .../drydock/storage/DrydockBlueprint.php | 2 +- .../configuration/HarbormasterBuildPlan.php | 2 +- .../nuance/editor/NuanceSourceEditor.php | 4 ++ .../NuanceManagementImportWorkflow.php | 50 ++++++++++++++ .../management/NuanceManagementWorkflow.php | 67 +++++++++++++++++++ .../nuance/query/NuanceSourceQuery.php | 22 ++++-- .../nuance/query/NuanceSourceSearchEngine.php | 11 ++- .../nuance/storage/NuanceSource.php | 16 ++++- .../nuance/storage/NuanceSourceNameNgrams.php | 18 +++++ .../storage/PhabricatorOwnersPackage.php | 2 +- .../workers/PhabricatorTriggerDaemon.php | 7 +- 21 files changed, 241 insertions(+), 17 deletions(-) create mode 120000 bin/nuance create mode 100644 resources/sql/autopatches/20160308.nuance.03.sourcen.sql create mode 100644 resources/sql/autopatches/20160308.nuance.04.sourcei.php create mode 100644 resources/sql/autopatches/20160308.nuance.05.sourcename.sql create mode 100755 scripts/setup/manage_nuance.php create mode 100644 src/applications/nuance/management/NuanceManagementImportWorkflow.php create mode 100644 src/applications/nuance/management/NuanceManagementWorkflow.php create mode 100644 src/applications/nuance/storage/NuanceSourceNameNgrams.php diff --git a/bin/nuance b/bin/nuance new file mode 120000 index 0000000000..c2cf50a211 --- /dev/null +++ b/bin/nuance @@ -0,0 +1 @@ +../scripts/setup/manage_nuance.php \ No newline at end of file diff --git a/resources/sql/autopatches/20160308.nuance.03.sourcen.sql b/resources/sql/autopatches/20160308.nuance.03.sourcen.sql new file mode 100644 index 0000000000..42ec4b87eb --- /dev/null +++ b/resources/sql/autopatches/20160308.nuance.03.sourcen.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_nuance.nuance_sourcename_ngrams ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + objectID INT UNSIGNED NOT NULL, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}, + KEY `key_object` (objectID), + KEY `key_ngram` (ngram, objectID) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20160308.nuance.04.sourcei.php b/resources/sql/autopatches/20160308.nuance.04.sourcei.php new file mode 100644 index 0000000000..eb0d1da113 --- /dev/null +++ b/resources/sql/autopatches/20160308.nuance.04.sourcei.php @@ -0,0 +1,11 @@ +getPHID(), + array( + 'force' => true, + )); +} diff --git a/resources/sql/autopatches/20160308.nuance.05.sourcename.sql b/resources/sql/autopatches/20160308.nuance.05.sourcename.sql new file mode 100644 index 0000000000..a2b70c683f --- /dev/null +++ b/resources/sql/autopatches/20160308.nuance.05.sourcename.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_nuance.nuance_source + CHANGE name name VARCHAR(255) NOT NULL COLLATE {$COLLATE_SORT}; diff --git a/scripts/setup/manage_nuance.php b/scripts/setup/manage_nuance.php new file mode 100755 index 0000000000..ebf312305a --- /dev/null +++ b/scripts/setup/manage_nuance.php @@ -0,0 +1,21 @@ +#!/usr/bin/env php +setTagline(pht('manage Nuance')); +$args->setSynopsis(<<parseStandardArguments(); + +$workflows = id(new PhutilClassMapQuery()) + ->setAncestorClass('NuanceManagementWorkflow') + ->execute(); +$workflows[] = new PhutilHelpArgumentWorkflow(); +$args->parseWorkflows($workflows); diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 2a49fd0979..7e49233e5f 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1434,6 +1434,8 @@ phutil_register_library_map(array( 'NuanceItemTransactionComment' => 'applications/nuance/storage/NuanceItemTransactionComment.php', 'NuanceItemTransactionQuery' => 'applications/nuance/query/NuanceItemTransactionQuery.php', 'NuanceItemViewController' => 'applications/nuance/controller/NuanceItemViewController.php', + 'NuanceManagementImportWorkflow' => 'applications/nuance/management/NuanceManagementImportWorkflow.php', + 'NuanceManagementWorkflow' => 'applications/nuance/management/NuanceManagementWorkflow.php', 'NuancePhabricatorFormSourceDefinition' => 'applications/nuance/source/NuancePhabricatorFormSourceDefinition.php', 'NuanceQuery' => 'applications/nuance/query/NuanceQuery.php', 'NuanceQueue' => 'applications/nuance/storage/NuanceQueue.php', @@ -1473,6 +1475,7 @@ phutil_register_library_map(array( 'NuanceSourceEditor' => 'applications/nuance/editor/NuanceSourceEditor.php', 'NuanceSourceListController' => 'applications/nuance/controller/NuanceSourceListController.php', 'NuanceSourceManageCapability' => 'applications/nuance/capability/NuanceSourceManageCapability.php', + 'NuanceSourceNameNgrams' => 'applications/nuance/storage/NuanceSourceNameNgrams.php', 'NuanceSourcePHIDType' => 'applications/nuance/phid/NuanceSourcePHIDType.php', 'NuanceSourceQuery' => 'applications/nuance/query/NuanceSourceQuery.php', 'NuanceSourceSearchEngine' => 'applications/nuance/query/NuanceSourceSearchEngine.php', @@ -5682,6 +5685,8 @@ phutil_register_library_map(array( 'NuanceItemTransactionComment' => 'PhabricatorApplicationTransactionComment', 'NuanceItemTransactionQuery' => 'PhabricatorApplicationTransactionQuery', 'NuanceItemViewController' => 'NuanceController', + 'NuanceManagementImportWorkflow' => 'NuanceManagementWorkflow', + 'NuanceManagementWorkflow' => 'PhabricatorManagementWorkflow', 'NuancePhabricatorFormSourceDefinition' => 'NuanceSourceDefinition', 'NuanceQuery' => 'PhabricatorCursorPagedPolicyAwareQuery', 'NuanceQueue' => array( @@ -5721,6 +5726,7 @@ phutil_register_library_map(array( 'NuanceDAO', 'PhabricatorApplicationTransactionInterface', 'PhabricatorPolicyInterface', + 'PhabricatorNgramsInterface', ), 'NuanceSourceActionController' => 'NuanceController', 'NuanceSourceController' => 'NuanceController', @@ -5733,6 +5739,7 @@ phutil_register_library_map(array( 'NuanceSourceEditor' => 'PhabricatorApplicationTransactionEditor', 'NuanceSourceListController' => 'NuanceSourceController', 'NuanceSourceManageCapability' => 'PhabricatorPolicyCapability', + 'NuanceSourceNameNgrams' => 'PhabricatorSearchNgrams', 'NuanceSourcePHIDType' => 'PhabricatorPHIDType', 'NuanceSourceQuery' => 'NuanceQuery', 'NuanceSourceSearchEngine' => 'PhabricatorApplicationSearchEngine', diff --git a/src/applications/almanac/storage/AlmanacDevice.php b/src/applications/almanac/storage/AlmanacDevice.php index f46b03600f..6c7f3cb57f 100644 --- a/src/applications/almanac/storage/AlmanacDevice.php +++ b/src/applications/almanac/storage/AlmanacDevice.php @@ -246,7 +246,7 @@ final class AlmanacDevice } -/* -( PhabricatorNgramInterface )------------------------------------------ */ +/* -( PhabricatorNgramsInterface )----------------------------------------- */ public function newNgrams() { diff --git a/src/applications/almanac/storage/AlmanacNamespace.php b/src/applications/almanac/storage/AlmanacNamespace.php index 6a3baca637..4bbb4d4090 100644 --- a/src/applications/almanac/storage/AlmanacNamespace.php +++ b/src/applications/almanac/storage/AlmanacNamespace.php @@ -210,7 +210,7 @@ final class AlmanacNamespace } -/* -( PhabricatorNgramInterface )------------------------------------------ */ +/* -( PhabricatorNgramsInterface )----------------------------------------- */ public function newNgrams() { diff --git a/src/applications/almanac/storage/AlmanacNetwork.php b/src/applications/almanac/storage/AlmanacNetwork.php index 064b612999..2f37ab4778 100644 --- a/src/applications/almanac/storage/AlmanacNetwork.php +++ b/src/applications/almanac/storage/AlmanacNetwork.php @@ -116,7 +116,7 @@ final class AlmanacNetwork } -/* -( PhabricatorNgramInterface )------------------------------------------ */ +/* -( PhabricatorNgramsInterface )----------------------------------------- */ public function newNgrams() { diff --git a/src/applications/almanac/storage/AlmanacService.php b/src/applications/almanac/storage/AlmanacService.php index b280de9ba8..37c3a44ba6 100644 --- a/src/applications/almanac/storage/AlmanacService.php +++ b/src/applications/almanac/storage/AlmanacService.php @@ -251,7 +251,7 @@ final class AlmanacService } -/* -( PhabricatorNgramInterface )------------------------------------------ */ +/* -( PhabricatorNgramsInterface )----------------------------------------- */ public function newNgrams() { diff --git a/src/applications/drydock/storage/DrydockBlueprint.php b/src/applications/drydock/storage/DrydockBlueprint.php index a7deb73cc3..87ea777f72 100644 --- a/src/applications/drydock/storage/DrydockBlueprint.php +++ b/src/applications/drydock/storage/DrydockBlueprint.php @@ -350,7 +350,7 @@ final class DrydockBlueprint extends DrydockDAO } -/* -( PhabricatorNgramInterface )------------------------------------------ */ +/* -( PhabricatorNgramsInterface )----------------------------------------- */ public function newNgrams() { diff --git a/src/applications/harbormaster/storage/configuration/HarbormasterBuildPlan.php b/src/applications/harbormaster/storage/configuration/HarbormasterBuildPlan.php index 8d706657ef..2a101ca355 100644 --- a/src/applications/harbormaster/storage/configuration/HarbormasterBuildPlan.php +++ b/src/applications/harbormaster/storage/configuration/HarbormasterBuildPlan.php @@ -197,7 +197,7 @@ final class HarbormasterBuildPlan extends HarbormasterDAO } -/* -( PhabricatorNgramInterface )------------------------------------------ */ +/* -( PhabricatorNgramsInterface )----------------------------------------- */ public function newNgrams() { diff --git a/src/applications/nuance/editor/NuanceSourceEditor.php b/src/applications/nuance/editor/NuanceSourceEditor.php index 233b2ae163..5fbc02b962 100644 --- a/src/applications/nuance/editor/NuanceSourceEditor.php +++ b/src/applications/nuance/editor/NuanceSourceEditor.php @@ -11,6 +11,10 @@ final class NuanceSourceEditor return pht('Nuance Sources'); } + protected function supportsSearch() { + return true; + } + public function getTransactionTypes() { $types = parent::getTransactionTypes(); diff --git a/src/applications/nuance/management/NuanceManagementImportWorkflow.php b/src/applications/nuance/management/NuanceManagementImportWorkflow.php new file mode 100644 index 0000000000..c8050ec1fc --- /dev/null +++ b/src/applications/nuance/management/NuanceManagementImportWorkflow.php @@ -0,0 +1,50 @@ +setName('import') + ->setExamples('**import** [__options__]') + ->setSynopsis(pht('Import data from a source.')) + ->setArguments( + array( + array( + 'name' => 'source', + 'param' => 'source', + 'help' => pht('Choose which source to import.'), + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $source = $this->loadSource($args, 'source'); + + $definition = $source->getDefinition() + ->setViewer($this->getViewer()) + ->setSource($source); + + if (!$definition->hasImportCursors()) { + throw new PhutilArgumentUsageException( + pht( + 'This source ("%s") does not expose import cursors.', + $source->getName())); + } + + $cursors = $definition->getImportCursors(); + if (!$cursors) { + throw new PhutilArgumentUsageException( + pht( + 'This source ("%s") does not have any import cursors.', + $source->getName())); + } + + echo tsprintf( + "%s\n", + pht('OK, but actual importing is not implemented yet.')); + + return 0; + } + +} diff --git a/src/applications/nuance/management/NuanceManagementWorkflow.php b/src/applications/nuance/management/NuanceManagementWorkflow.php new file mode 100644 index 0000000000..f6dbfe6e09 --- /dev/null +++ b/src/applications/nuance/management/NuanceManagementWorkflow.php @@ -0,0 +1,67 @@ +getArg($key); + if (!strlen($source)) { + throw new PhutilArgumentUsageException( + pht( + 'Specify a source with %s.', + '--'.$key)); + } + + $query = id(new NuanceSourceQuery()) + ->setViewer($this->getViewer()) + ->setRaisePolicyExceptions(true); + + $type_unknown = PhabricatorPHIDConstants::PHID_TYPE_UNKNOWN; + + if (ctype_digit($source)) { + $kind = 'id'; + $query->withIDs(array($source)); + } else if (phid_get_type($source) !== $type_unknown) { + $kind = 'phid'; + $query->withPHIDs($source); + } else { + $kind = 'name'; + $query->withNameNgrams($source); + } + + $sources = $query->execute(); + + if (!$sources) { + switch ($kind) { + case 'id': + $message = pht( + 'No source exists with ID "%s".', + $source); + break; + case 'phid': + $message = pht( + 'No source exists with PHID "%s".', + $source); + break; + default: + $message = pht( + 'No source exists with a name matching "%s".', + $source); + break; + } + + throw new PhutilArgumentUsageException($message); + } else if (count($sources) > 1) { + $message = pht( + 'More than one source matches "%s". Choose a narrower query, or '. + 'use an ID or PHID to select a source. Matching sources: %s.', + $source, + implode(', ', mpull($sources, 'getName'))); + + throw new PhutilArgumentUsageException($message); + } + + return head($sources); + } + +} diff --git a/src/applications/nuance/query/NuanceSourceQuery.php b/src/applications/nuance/query/NuanceSourceQuery.php index 7bde9440e2..907d9c314f 100644 --- a/src/applications/nuance/query/NuanceSourceQuery.php +++ b/src/applications/nuance/query/NuanceSourceQuery.php @@ -34,10 +34,20 @@ final class NuanceSourceQuery return $this; } + public function withNameNgrams($ngrams) { + return $this->withNgramsConstraint( + new NuanceSourceNameNgrams(), + $ngrams); + } + public function newResultObject() { return new NuanceSource(); } + protected function getPrimaryTableAlias() { + return 'source'; + } + protected function loadPage() { return $this->loadStandardPage($this->newResultObject()); } @@ -65,28 +75,28 @@ final class NuanceSourceQuery if ($this->types !== null) { $where[] = qsprintf( $conn, - 'type IN (%Ls)', + 'source.type IN (%Ls)', $this->types); } if ($this->ids !== null) { $where[] = qsprintf( $conn, - 'id IN (%Ld)', + 'source.id IN (%Ld)', $this->ids); } if ($this->phids !== null) { $where[] = qsprintf( $conn, - 'phid IN (%Ls)', + 'source.phid IN (%Ls)', $this->phids); } if ($this->isDisabled !== null) { $where[] = qsprintf( $conn, - 'isDisabled = %d', + 'source.isDisabled = %d', (int)$this->isDisabled); } @@ -106,7 +116,7 @@ final class NuanceSourceQuery } else { $where[] = qsprintf( $conn, - 'type IN (%Ls)', + 'source.type IN (%Ls)', $cursor_types); } } else { @@ -115,7 +125,7 @@ final class NuanceSourceQuery } else { $where[] = qsprintf( $conn, - 'type NOT IN (%Ls)', + 'source.type NOT IN (%Ls)', $cursor_types); } } diff --git a/src/applications/nuance/query/NuanceSourceSearchEngine.php b/src/applications/nuance/query/NuanceSourceSearchEngine.php index 2991c69c5f..44f131aa1b 100644 --- a/src/applications/nuance/query/NuanceSourceSearchEngine.php +++ b/src/applications/nuance/query/NuanceSourceSearchEngine.php @@ -18,11 +18,20 @@ final class NuanceSourceSearchEngine protected function buildQueryFromParameters(array $map) { $query = $this->newQuery(); + if ($map['match'] !== null) { + $query->withNameNgrams($map['match']); + } + return $query; } protected function buildCustomSearchFields() { - return array(); + return array( + id(new PhabricatorSearchTextField()) + ->setLabel(pht('Name Contains')) + ->setKey('match') + ->setDescription(pht('Search for sources by name substring.')), + ); } protected function getURI($path) { diff --git a/src/applications/nuance/storage/NuanceSource.php b/src/applications/nuance/storage/NuanceSource.php index 52da800e4f..9f60d03b51 100644 --- a/src/applications/nuance/storage/NuanceSource.php +++ b/src/applications/nuance/storage/NuanceSource.php @@ -3,7 +3,8 @@ final class NuanceSource extends NuanceDAO implements PhabricatorApplicationTransactionInterface, - PhabricatorPolicyInterface { + PhabricatorPolicyInterface, + PhabricatorNgramsInterface { protected $name; protected $type; @@ -23,7 +24,7 @@ final class NuanceSource extends NuanceDAO 'data' => self::SERIALIZATION_JSON, ), self::CONFIG_COLUMN_SCHEMA => array( - 'name' => 'text255?', + 'name' => 'sort255', 'type' => 'text32', 'mailKey' => 'bytes20', 'isDisabled' => 'bool', @@ -132,4 +133,15 @@ final class NuanceSource extends NuanceDAO return null; } + +/* -( PhabricatorNgramsInterface )----------------------------------------- */ + + + public function newNgrams() { + return array( + id(new NuanceSourceNameNgrams()) + ->setValue($this->getName()), + ); + } + } diff --git a/src/applications/nuance/storage/NuanceSourceNameNgrams.php b/src/applications/nuance/storage/NuanceSourceNameNgrams.php new file mode 100644 index 0000000000..8bff4b8a2a --- /dev/null +++ b/src/applications/nuance/storage/NuanceSourceNameNgrams.php @@ -0,0 +1,18 @@ +nuanceCursors) { $source = array_pop($this->nuanceSources); - $cursors = $source->getImportCursors(); + + $definition = $source->getDefinition() + ->setViewer($this->getViewer()) + ->setSource($source); + + $cursors = $definition->getImportCursors(); $this->nuanceCursors = array_reverse($cursors); }