revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-22 14:52:41 +01:00
Code Issues Releases Wiki Activity

Optimize reindex_everything.php

Browse source 
Summary:
We have two troubles with this script:

# Our revisions and commits don't fit in the memory. (Our tasks do :-).)
# Reindexing revisions is slow.

Test Plan: Ran it.

Reviewers: epriestley

Reviewed By: epriestley

CC: aran, Korvin

Differential Revision: https://secure.phabricator.com/D3483
This commit is contained in:
vrana 2012-09-12 14:53:30 -07:00
parent 3c2cb13153
commit 8e7ae7b33a
3 changed files with 19 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
scripts/search/reindex_everything.php
View file

@ -2,7 +2,7 @@
<?php <?php
/* /*
* Copyright 2011 Facebook, Inc. * Copyright 2012 Facebook, Inc.
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -23,34 +23,28 @@ require_once $root.'/scripts/__init_script__.php';
// TODO: Get rid of this script eventually, once this stuff is better-formalized // TODO: Get rid of this script eventually, once this stuff is better-formalized
// in Timeline consumers. // in Timeline consumers.
echo "Loading revisions...\n"; echo "Reindexing revisions...\n";
$revs = id(new DifferentialRevision())->loadAll(); $revs = new LiskMigrationIterator(new DifferentialRevision());
$count = count($revs);
echo "Reindexing {$count} revisions";
foreach ($revs as $rev) { foreach ($revs as $rev) {
PhabricatorSearchDifferentialIndexer::indexRevision($rev); PhabricatorSearchDifferentialIndexer::indexRevision($rev);
echo '.'; echo '.';
} }
echo "\n"; echo "\n";
echo "Loading commits...\n"; echo "Reindexing commits...\n";
$commits = id(new PhabricatorRepositoryCommit())->loadAll(); $commits = new LiskMigrationIterator(new PhabricatorRepositoryCommit());
$count = count($commits);
echo "Reindexing {$count} commits";
foreach ($commits as $commit) { foreach ($commits as $commit) {
PhabricatorSearchCommitIndexer::indexCommit($commit); PhabricatorSearchCommitIndexer::indexCommit($commit);
echo '.'; echo '.';
} }
echo "\n"; echo "\n";
echo "Loading tasks...\n"; echo "Reindexing tasks...\n";
$tasks = id(new ManiphestTask())->loadAll(); $tasks = new LiskMigrationIterator(new ManiphestTask());
$count = count($tasks);
echo "Reindexing {$count} tasks";
foreach ($tasks as $task) { foreach ($tasks as $task) {
PhabricatorSearchManiphestIndexer::indexTask($task); PhabricatorSearchManiphestIndexer::indexTask($task);
echo '.'; echo '.';
} }
echo "\n"; echo "\n";
echo "Done.\n";
include dirname(__FILE__).'/reindex_all_users.php';

12
src/applications/search/index/indexer/PhabricatorSearchDifferentialIndexer.php
View file

@ -52,13 +52,13 @@ final class PhabricatorSearchDifferentialIndexer
time()); time());
} }
$comments = id(new DifferentialComment())->loadAllWhere( $comments = $rev->loadRelatives(new DifferentialComment(), 'revisionID');
'revisionID = %d',
$rev->getID());
$inlines = id(new DifferentialInlineComment())->loadAllWhere( $inlines = $rev->loadRelatives(
'revisionID = %d AND commentID IS NOT NULL', new DifferentialInlineComment(),
$rev->getID()); 'revisionID',
'getID',
'(commentID IS NOT NULL)');
$touches = array(); $touches = array();

6
src/infrastructure/storage/lisk/LiskMigrationIterator.php
View file

@ -33,9 +33,11 @@ final class LiskMigrationIterator extends PhutilBufferedIterator {
private $object; private $object;
private $cursor; private $cursor;
private $set;
public function __construct(LiskDAO $object) { public function __construct(LiskDAO $object) {
$this->object = $object; $this->set = new LiskDAOSet();
$this->object = $object->putInSet($this->set);
} }
protected function didRewind() { protected function didRewind() {
@ -47,6 +49,8 @@ final class LiskMigrationIterator extends PhutilBufferedIterator {
} }
protected function loadPage() { protected function loadPage() {
$this->set->clearSet();
$results = $this->object->loadAllWhere( $results = $this->object->loadAllWhere(
'id > %d ORDER BY id ASC LIMIT %d', 'id > %d ORDER BY id ASC LIMIT %d',
$this->cursor, $this->cursor,