revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-10 00:42:41 +01:00
Code Issues Releases Wiki Activity

Limit maximum size of Owners package queries

Browse source 
Summary:
Currently, a change may affect a very large number of paths. When we run the OwnersWorker on it, we'll execute a query which looks up packages for the paths. This may exceed "max_allowed_packet". Instead, break the list of paths into smaller chunks.

This is mostly to unblock r4nt / llvm, I'm going to add a more finessed approach to array_chunk() shortly.

Test Plan: Ran `reparse.php --owners` on a revision which affected packages, verified results are the same before and after the change. Set chunk size to 1, verified query results aggregated properly.

Reviewers: btrahan, jungejason, nh

Reviewed By: jungejason

CC: aran

Differential Revision: https://secure.phabricator.com/D2943
This commit is contained in:
epriestley 2012-07-09 10:51:46 -07:00
parent 1089a48d4a
commit 918ac5755c
1 changed files with 37 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
src/applications/owners/storage/PhabricatorOwnersPackage.php
View file

@ -142,40 +142,51 @@ final class PhabricatorOwnersPackage extends PhabricatorOwnersDAO {
$path = new PhabricatorOwnersPath();
$conn = $package->establishConnection('r');
$repository_clause = qsprintf($conn, 'AND p.repositoryPHID = %s',
$repository_clause = qsprintf(
$conn,
'AND p.repositoryPHID = %s',
$repository->getPHID());
$limit_clause = '';
if (!empty($limit)) {
$limit_clause = qsprintf($conn, 'LIMIT %d', $limit);
}
// NOTE: The list of $paths may be very large if we're coming from
// the OwnersWorker and processing, e.g., an SVN commit which created a new
// branch. Break it apart so that it will fit within 'max_allowed_packet',
// and then merge results in PHP.
$data = queryfx_all(
$conn,
'SELECT pkg.id FROM %T pkg JOIN %T p ON p.packageID = pkg.id
WHERE p.path IN (%Ls) %Q ORDER BY LENGTH(p.path) DESC %Q',
$package->getTableName(),
$path->getTableName(),
$paths,
$repository_clause,
$limit_clause);
$ids = array();
foreach (array_chunk($paths, 128) as $chunk) {
$rows = queryfx_all(
$conn,
'SELECT pkg.id id, LENGTH(p.path) len
FROM %T pkg JOIN %T p ON p.packageID = pkg.id
WHERE p.path IN (%Ls) %Q',
$package->getTableName(),
$path->getTableName(),
$chunk,
$repository_clause);
$ids = ipull($data, 'id');
if (empty($ids)) {
return array();
}
$order = array();
foreach ($ids as $id) {
if (empty($order[$id])) {
$order[$id] = true;
foreach ($rows as $row) {
$id = (int)$row['id'];
$len = (int)$row['len'];
if (isset($ids[$id])) {
$ids[$id] = max($len, $ids[$id]);
} else {
$ids[$id] = $len;
}
}
}
$packages = $package->loadAllWhere('id in (%Ld)', array_keys($order));
if (!$ids) {
return array();
}
$packages = array_select_keys($packages, array_keys($order));
arsort($ids);
if ($limit) {
$ids = array_slice($ids, 0, $limit, $preserve_keys = true);
}
$ids = array_keys($ids);
$packages = $package->loadAllWhere('id in (%Ld)', array_keys($ids));
$packages = array_select_keys($packages, array_keys($ids));
return $packages;
}