revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-27 01:02:42 +01:00
Code Issues Releases Wiki Activity

Limit maximum size of Owners package queries

Browse source 
Summary:
Currently, a change may affect a very large number of paths. When we run the OwnersWorker on it, we'll execute a query which looks up packages for the paths. This may exceed "max_allowed_packet". Instead, break the list of paths into smaller chunks.

This is mostly to unblock r4nt / llvm, I'm going to add a more finessed approach to array_chunk() shortly.

Test Plan: Ran `reparse.php --owners` on a revision which affected packages, verified results are the same before and after the change. Set chunk size to 1, verified query results aggregated properly.

Reviewers: btrahan, jungejason, nh

Reviewed By: jungejason

CC: aran

Differential Revision: https://secure.phabricator.com/D2943
This commit is contained in:
epriestley 2012-07-09 10:51:46 -07:00
parent 1089a48d4a
commit 918ac5755c
1 changed files with 37 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
src/applications/owners/storage/PhabricatorOwnersPackage.php
View file

@ -142,40 +142,51 @@ final class PhabricatorOwnersPackage extends PhabricatorOwnersDAO {
$path = new PhabricatorOwnersPath(); $path = new PhabricatorOwnersPath();
$conn = $package->establishConnection('r'); $conn = $package->establishConnection('r');
$repository_clause = qsprintf($conn, 'AND p.repositoryPHID = %s', $repository_clause = qsprintf(
$conn,
'AND p.repositoryPHID = %s',
$repository->getPHID()); $repository->getPHID());
$limit_clause = ''; // NOTE: The list of $paths may be very large if we're coming from
if (!empty($limit)) { // the OwnersWorker and processing, e.g., an SVN commit which created a new
$limit_clause = qsprintf($conn, 'LIMIT %d', $limit); // branch. Break it apart so that it will fit within 'max_allowed_packet',
} // and then merge results in PHP.
$data = queryfx_all( $ids = array();
$conn, foreach (array_chunk($paths, 128) as $chunk) {
'SELECT pkg.id FROM %T pkg JOIN %T p ON p.packageID = pkg.id $rows = queryfx_all(
WHERE p.path IN (%Ls) %Q ORDER BY LENGTH(p.path) DESC %Q', $conn,
$package->getTableName(), 'SELECT pkg.id id, LENGTH(p.path) len
$path->getTableName(), FROM %T pkg JOIN %T p ON p.packageID = pkg.id
$paths, WHERE p.path IN (%Ls) %Q',
$repository_clause, $package->getTableName(),
$limit_clause); $path->getTableName(),
$chunk,
$repository_clause);
$ids = ipull($data, 'id'); foreach ($rows as $row) {
$id = (int)$row['id'];
if (empty($ids)) { $len = (int)$row['len'];
return array(); if (isset($ids[$id])) {
} $ids[$id] = max($len, $ids[$id]);
} else {
$order = array(); $ids[$id] = $len;
foreach ($ids as $id) { }
if (empty($order[$id])) {
$order[$id] = true;
} }
} }
$packages = $package->loadAllWhere('id in (%Ld)', array_keys($order)); if (!$ids) {
return array();
}
$packages = array_select_keys($packages, array_keys($order)); arsort($ids);
if ($limit) {
$ids = array_slice($ids, 0, $limit, $preserve_keys = true);
}
$ids = array_keys($ids);
$packages = $package->loadAllWhere('id in (%Ld)', array_keys($ids));
$packages = array_select_keys($packages, array_keys($ids));
return $packages; return $packages;
} }