From b1703c8801f4448cf13666b3b8ef49674f791f9d Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 7 Sep 2017 12:14:05 -0700 Subject: [PATCH] Support Ferret engine in Phame Summary: Ref T12819. Mostly straightforward, with a couple of minor query modernization things. Test Plan: Indexed and searched for posts and blogs. Reviewers: chad Reviewed By: chad Maniphest Tasks: T12819 Differential Revision: https://secure.phabricator.com/D18565 --- .../20170907.ferret.13.blog.doc.sql | 9 +++++++ .../20170907.ferret.14.blog.field.sql | 8 ++++++ .../20170907.ferret.15.blog.ngrams.sql | 5 ++++ .../20170907.ferret.16.post.doc.sql | 9 +++++++ .../20170907.ferret.17.post.field.sql | 8 ++++++ .../20170907.ferret.18.post.ngrams.sql | 5 ++++ src/__phutil_library_map__.php | 6 +++++ .../phame/query/PhameBlogQuery.php | 12 ++++++--- .../phame/query/PhamePostQuery.php | 25 +++++++++++-------- .../phame/search/PhameBlogFerretEngine.php | 18 +++++++++++++ .../phame/search/PhamePostFerretEngine.php | 18 +++++++++++++ src/applications/phame/storage/PhameBlog.php | 11 +++++++- src/applications/phame/storage/PhamePost.php | 11 +++++++- 13 files changed, 129 insertions(+), 16 deletions(-) create mode 100644 resources/sql/autopatches/20170907.ferret.13.blog.doc.sql create mode 100644 resources/sql/autopatches/20170907.ferret.14.blog.field.sql create mode 100644 resources/sql/autopatches/20170907.ferret.15.blog.ngrams.sql create mode 100644 resources/sql/autopatches/20170907.ferret.16.post.doc.sql create mode 100644 resources/sql/autopatches/20170907.ferret.17.post.field.sql create mode 100644 resources/sql/autopatches/20170907.ferret.18.post.ngrams.sql create mode 100644 src/applications/phame/search/PhameBlogFerretEngine.php create mode 100644 src/applications/phame/search/PhamePostFerretEngine.php diff --git a/resources/sql/autopatches/20170907.ferret.13.blog.doc.sql b/resources/sql/autopatches/20170907.ferret.13.blog.doc.sql new file mode 100644 index 0000000000..d75232fae1 --- /dev/null +++ b/resources/sql/autopatches/20170907.ferret.13.blog.doc.sql @@ -0,0 +1,9 @@ +CREATE TABLE {$NAMESPACE}_phame.phame_blog_fdocument ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + objectPHID VARBINARY(64) NOT NULL, + isClosed BOOL NOT NULL, + authorPHID VARBINARY(64), + ownerPHID VARBINARY(64), + epochCreated INT UNSIGNED NOT NULL, + epochModified INT UNSIGNED NOT NULL +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20170907.ferret.14.blog.field.sql b/resources/sql/autopatches/20170907.ferret.14.blog.field.sql new file mode 100644 index 0000000000..9982914229 --- /dev/null +++ b/resources/sql/autopatches/20170907.ferret.14.blog.field.sql @@ -0,0 +1,8 @@ +CREATE TABLE {$NAMESPACE}_phame.phame_blog_ffield ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + documentID INT UNSIGNED NOT NULL, + fieldKey VARCHAR(4) NOT NULL COLLATE {$COLLATE_TEXT}, + rawCorpus LONGTEXT NOT NULL COLLATE {$COLLATE_SORT}, + termCorpus LONGTEXT NOT NULL COLLATE {$COLLATE_SORT}, + normalCorpus LONGTEXT NOT NULL COLLATE {$COLLATE_SORT} +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20170907.ferret.15.blog.ngrams.sql b/resources/sql/autopatches/20170907.ferret.15.blog.ngrams.sql new file mode 100644 index 0000000000..b20bb8fcbb --- /dev/null +++ b/resources/sql/autopatches/20170907.ferret.15.blog.ngrams.sql @@ -0,0 +1,5 @@ +CREATE TABLE {$NAMESPACE}_phame.phame_blog_fngrams ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + documentID INT UNSIGNED NOT NULL, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT} +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20170907.ferret.16.post.doc.sql b/resources/sql/autopatches/20170907.ferret.16.post.doc.sql new file mode 100644 index 0000000000..9f9155aa49 --- /dev/null +++ b/resources/sql/autopatches/20170907.ferret.16.post.doc.sql @@ -0,0 +1,9 @@ +CREATE TABLE {$NAMESPACE}_phame.phame_post_fdocument ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + objectPHID VARBINARY(64) NOT NULL, + isClosed BOOL NOT NULL, + authorPHID VARBINARY(64), + ownerPHID VARBINARY(64), + epochCreated INT UNSIGNED NOT NULL, + epochModified INT UNSIGNED NOT NULL +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20170907.ferret.17.post.field.sql b/resources/sql/autopatches/20170907.ferret.17.post.field.sql new file mode 100644 index 0000000000..26d729d05d --- /dev/null +++ b/resources/sql/autopatches/20170907.ferret.17.post.field.sql @@ -0,0 +1,8 @@ +CREATE TABLE {$NAMESPACE}_phame.phame_post_ffield ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + documentID INT UNSIGNED NOT NULL, + fieldKey VARCHAR(4) NOT NULL COLLATE {$COLLATE_TEXT}, + rawCorpus LONGTEXT NOT NULL COLLATE {$COLLATE_SORT}, + termCorpus LONGTEXT NOT NULL COLLATE {$COLLATE_SORT}, + normalCorpus LONGTEXT NOT NULL COLLATE {$COLLATE_SORT} +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20170907.ferret.18.post.ngrams.sql b/resources/sql/autopatches/20170907.ferret.18.post.ngrams.sql new file mode 100644 index 0000000000..18e534e948 --- /dev/null +++ b/resources/sql/autopatches/20170907.ferret.18.post.ngrams.sql @@ -0,0 +1,5 @@ +CREATE TABLE {$NAMESPACE}_phame.phame_post_fngrams ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + documentID INT UNSIGNED NOT NULL, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT} +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 266993bfb7..7a26123482 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -4371,6 +4371,7 @@ phutil_register_library_map(array( 'PhameBlogEditEngine' => 'applications/phame/editor/PhameBlogEditEngine.php', 'PhameBlogEditor' => 'applications/phame/editor/PhameBlogEditor.php', 'PhameBlogFeedController' => 'applications/phame/controller/blog/PhameBlogFeedController.php', + 'PhameBlogFerretEngine' => 'applications/phame/search/PhameBlogFerretEngine.php', 'PhameBlogFullDomainTransaction' => 'applications/phame/xaction/PhameBlogFullDomainTransaction.php', 'PhameBlogFulltextEngine' => 'applications/phame/search/PhameBlogFulltextEngine.php', 'PhameBlogHeaderImageTransaction' => 'applications/phame/xaction/PhameBlogHeaderImageTransaction.php', @@ -4411,6 +4412,7 @@ phutil_register_library_map(array( 'PhamePostEditController' => 'applications/phame/controller/post/PhamePostEditController.php', 'PhamePostEditEngine' => 'applications/phame/editor/PhamePostEditEngine.php', 'PhamePostEditor' => 'applications/phame/editor/PhamePostEditor.php', + 'PhamePostFerretEngine' => 'applications/phame/search/PhamePostFerretEngine.php', 'PhamePostFulltextEngine' => 'applications/phame/search/PhamePostFulltextEngine.php', 'PhamePostHeaderImageTransaction' => 'applications/phame/xaction/PhamePostHeaderImageTransaction.php', 'PhamePostHeaderPictureController' => 'applications/phame/controller/post/PhamePostHeaderPictureController.php', @@ -9995,6 +9997,7 @@ phutil_register_library_map(array( 'PhabricatorApplicationTransactionInterface', 'PhabricatorConduitResultInterface', 'PhabricatorFulltextInterface', + 'PhabricatorFerretInterface', ), 'PhameBlog404Controller' => 'PhameLiveController', 'PhameBlogArchiveController' => 'PhameBlogController', @@ -10007,6 +10010,7 @@ phutil_register_library_map(array( 'PhameBlogEditEngine' => 'PhabricatorEditEngine', 'PhameBlogEditor' => 'PhabricatorApplicationTransactionEditor', 'PhameBlogFeedController' => 'PhameBlogController', + 'PhameBlogFerretEngine' => 'PhabricatorFerretEngine', 'PhameBlogFullDomainTransaction' => 'PhameBlogTransactionType', 'PhameBlogFulltextEngine' => 'PhabricatorFulltextEngine', 'PhameBlogHeaderImageTransaction' => 'PhameBlogTransactionType', @@ -10050,6 +10054,7 @@ phutil_register_library_map(array( 'PhabricatorTokenReceiverInterface', 'PhabricatorConduitResultInterface', 'PhabricatorFulltextInterface', + 'PhabricatorFerretInterface', ), 'PhamePostArchiveController' => 'PhamePostController', 'PhamePostBlogTransaction' => 'PhamePostTransactionType', @@ -10059,6 +10064,7 @@ phutil_register_library_map(array( 'PhamePostEditController' => 'PhamePostController', 'PhamePostEditEngine' => 'PhabricatorEditEngine', 'PhamePostEditor' => 'PhabricatorApplicationTransactionEditor', + 'PhamePostFerretEngine' => 'PhabricatorFerretEngine', 'PhamePostFulltextEngine' => 'PhabricatorFulltextEngine', 'PhamePostHeaderImageTransaction' => 'PhamePostTransactionType', 'PhamePostHeaderPictureController' => 'PhamePostController', diff --git a/src/applications/phame/query/PhameBlogQuery.php b/src/applications/phame/query/PhameBlogQuery.php index bd34255bd0..b4018c78eb 100644 --- a/src/applications/phame/query/PhameBlogQuery.php +++ b/src/applications/phame/query/PhameBlogQuery.php @@ -55,28 +55,28 @@ final class PhameBlogQuery extends PhabricatorCursorPagedPolicyAwareQuery { if ($this->statuses !== null) { $where[] = qsprintf( $conn, - 'status IN (%Ls)', + 'b.status IN (%Ls)', $this->statuses); } if ($this->ids !== null) { $where[] = qsprintf( $conn, - 'id IN (%Ls)', + 'b.id IN (%Ls)', $this->ids); } if ($this->phids !== null) { $where[] = qsprintf( $conn, - 'phid IN (%Ls)', + 'b.phid IN (%Ls)', $this->phids); } if ($this->domain !== null) { $where[] = qsprintf( $conn, - 'domain = %s', + 'b.domain = %s', $this->domain); } @@ -143,4 +143,8 @@ final class PhameBlogQuery extends PhabricatorCursorPagedPolicyAwareQuery { return null; } + protected function getPrimaryTableAlias() { + return 'b'; + } + } diff --git a/src/applications/phame/query/PhamePostQuery.php b/src/applications/phame/query/PhamePostQuery.php index 357418cfb3..85ef470cea 100644 --- a/src/applications/phame/query/PhamePostQuery.php +++ b/src/applications/phame/query/PhamePostQuery.php @@ -106,45 +106,45 @@ final class PhamePostQuery extends PhabricatorCursorPagedPolicyAwareQuery { protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { $where = parent::buildWhereClauseParts($conn); - if ($this->ids) { + if ($this->ids !== null) { $where[] = qsprintf( $conn, - 'id IN (%Ld)', + 'p.id IN (%Ld)', $this->ids); } - if ($this->phids) { + if ($this->phids !== null) { $where[] = qsprintf( $conn, - 'phid IN (%Ls)', + 'p.phid IN (%Ls)', $this->phids); } - if ($this->bloggerPHIDs) { + if ($this->bloggerPHIDs !== null) { $where[] = qsprintf( $conn, - 'bloggerPHID IN (%Ls)', + 'p.bloggerPHID IN (%Ls)', $this->bloggerPHIDs); } - if ($this->visibility) { + if ($this->visibility !== null) { $where[] = qsprintf( $conn, - 'visibility IN (%Ld)', + 'p.visibility IN (%Ld)', $this->visibility); } if ($this->publishedAfter !== null) { $where[] = qsprintf( $conn, - 'datePublished > %d', + 'p.datePublished > %d', $this->publishedAfter); } if ($this->blogPHIDs !== null) { $where[] = qsprintf( $conn, - 'blogPHID in (%Ls)', + 'p.blogPHID in (%Ls)', $this->blogPHIDs); } @@ -163,6 +163,7 @@ final class PhamePostQuery extends PhabricatorCursorPagedPolicyAwareQuery { public function getOrderableColumns() { return parent::getOrderableColumns() + array( 'datePublished' => array( + 'table' => $this->getPrimaryTableAlias(), 'column' => 'datePublished', 'type' => 'int', 'reverse' => false, @@ -186,4 +187,8 @@ final class PhamePostQuery extends PhabricatorCursorPagedPolicyAwareQuery { return null; } + protected function getPrimaryTableAlias() { + return 'p'; + } + } diff --git a/src/applications/phame/search/PhameBlogFerretEngine.php b/src/applications/phame/search/PhameBlogFerretEngine.php new file mode 100644 index 0000000000..76901f915c --- /dev/null +++ b/src/applications/phame/search/PhameBlogFerretEngine.php @@ -0,0 +1,18 @@ +