revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-22 06:42:42 +01:00
Code Issues Releases Wiki Activity

Optimize reindex_everything.php

Browse source 
Summary:
We have two troubles with this script:

# Our revisions and commits don't fit in the memory. (Our tasks do :-).)
# Reindexing revisions is slow.

Test Plan: Ran it.

Reviewers: epriestley

Reviewed By: epriestley

CC: aran, Korvin

Differential Revision: https://secure.phabricator.com/D3483
This commit is contained in:
vrana 2012-09-12 14:53:30 -07:00
parent 3c2cb13153
commit 8e7ae7b33a
3 changed files with 19 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
scripts/search/reindex_everything.php
View file

@ -2,7 +2,7 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -23,34 +23,28 @@ require_once $root.'/scripts/__init_script__.php';
// TODO: Get rid of this script eventually, once this stuff is better-formalized
// in Timeline consumers.
echo "Loading revisions...\n";
$revs = id(new DifferentialRevision())->loadAll();
$count = count($revs);
echo "Reindexing {$count} revisions";
echo "Reindexing revisions...\n";
$revs = new LiskMigrationIterator(new DifferentialRevision());
foreach ($revs as $rev) {
PhabricatorSearchDifferentialIndexer::indexRevision($rev);
echo '.';
}
echo "\n";
echo "Loading commits...\n";
$commits = id(new PhabricatorRepositoryCommit())->loadAll();
$count = count($commits);
echo "Reindexing {$count} commits";
echo "Reindexing commits...\n";
$commits = new LiskMigrationIterator(new PhabricatorRepositoryCommit());
foreach ($commits as $commit) {
PhabricatorSearchCommitIndexer::indexCommit($commit);
echo '.';
}
echo "\n";
echo "Loading tasks...\n";
$tasks = id(new ManiphestTask())->loadAll();
$count = count($tasks);
echo "Reindexing {$count} tasks";
echo "Reindexing tasks...\n";
$tasks = new LiskMigrationIterator(new ManiphestTask());
foreach ($tasks as $task) {
PhabricatorSearchManiphestIndexer::indexTask($task);
echo '.';
}
echo "\n";
echo "Done.\n";
include dirname(__FILE__).'/reindex_all_users.php';

12
src/applications/search/index/indexer/PhabricatorSearchDifferentialIndexer.php
View file

@ -52,13 +52,13 @@ final class PhabricatorSearchDifferentialIndexer
time());
}
$comments = id(new DifferentialComment())->loadAllWhere(
'revisionID = %d',
$rev->getID());
$comments = $rev->loadRelatives(new DifferentialComment(), 'revisionID');
$inlines = id(new DifferentialInlineComment())->loadAllWhere(
'revisionID = %d AND commentID IS NOT NULL',
$rev->getID());
$inlines = $rev->loadRelatives(
new DifferentialInlineComment(),
'revisionID',
'getID',
'(commentID IS NOT NULL)');
$touches = array();

6
src/infrastructure/storage/lisk/LiskMigrationIterator.php
View file

@ -33,9 +33,11 @@ final class LiskMigrationIterator extends PhutilBufferedIterator {
private $object;
private $cursor;
private $set;
public function __construct(LiskDAO $object) {
$this->object = $object;
$this->set = new LiskDAOSet();
$this->object = $object->putInSet($this->set);
}
protected function didRewind() {
@ -47,6 +49,8 @@ final class LiskMigrationIterator extends PhutilBufferedIterator {
}
protected function loadPage() {
$this->set->clearSet();
$results = $this->object->loadAllWhere(
'id > %d ORDER BY id ASC LIMIT %d',
$this->cursor,