1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-12-23 22:10:55 +01:00

Add "bin/storage optimize" to run OPTIMIZE TABLE on everything

Summary:
Even with `innodb_file_per_table` enabled, individual table files on disk don't normally shrink.

For most tables, like `maniphest_task`, this is fine, since the data in the table normally never shrinks, or only shinks a tiny amount.

However, some tables (like the "worker" and "daemon" tables) grow very large during a huge import but most of the data is later deleted by garbage collection. In these cases, this lost space can be reclaimed by running `OPTIMIZE TABLE` on the tables.

Add a script to `OPTIMIZE TABLE` every table.

My primary goal here is just to reduce storage pressure on `db001` since there are a couple of "import the linux kernel" installs on that host wasting a bunch of space. We're not in any trouble, but this should buy us a good chunk of headroom.

Test Plan: Ran `bin/storage optimize` locally and manually ran `OPTIMIZE TABLE` in production, saw tables get optimized.

Reviewers: chad

Reviewed By: chad

Subscribers: cspeckmim

Differential Revision: https://secure.phabricator.com/D17640
This commit is contained in:
epriestley 2017-04-08 10:23:30 -07:00
parent 7707685733
commit d1421bc3a1
2 changed files with 88 additions and 0 deletions

View file

@ -3940,6 +3940,7 @@ phutil_register_library_map(array(
'PhabricatorStorageManagementDatabasesWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDatabasesWorkflow.php', 'PhabricatorStorageManagementDatabasesWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDatabasesWorkflow.php',
'PhabricatorStorageManagementDestroyWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php', 'PhabricatorStorageManagementDestroyWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php',
'PhabricatorStorageManagementDumpWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php', 'PhabricatorStorageManagementDumpWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php',
'PhabricatorStorageManagementOptimizeWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementOptimizeWorkflow.php',
'PhabricatorStorageManagementPartitionWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementPartitionWorkflow.php', 'PhabricatorStorageManagementPartitionWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementPartitionWorkflow.php',
'PhabricatorStorageManagementProbeWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementProbeWorkflow.php', 'PhabricatorStorageManagementProbeWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementProbeWorkflow.php',
'PhabricatorStorageManagementQuickstartWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php', 'PhabricatorStorageManagementQuickstartWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php',
@ -9286,6 +9287,7 @@ phutil_register_library_map(array(
'PhabricatorStorageManagementDatabasesWorkflow' => 'PhabricatorStorageManagementWorkflow', 'PhabricatorStorageManagementDatabasesWorkflow' => 'PhabricatorStorageManagementWorkflow',
'PhabricatorStorageManagementDestroyWorkflow' => 'PhabricatorStorageManagementWorkflow', 'PhabricatorStorageManagementDestroyWorkflow' => 'PhabricatorStorageManagementWorkflow',
'PhabricatorStorageManagementDumpWorkflow' => 'PhabricatorStorageManagementWorkflow', 'PhabricatorStorageManagementDumpWorkflow' => 'PhabricatorStorageManagementWorkflow',
'PhabricatorStorageManagementOptimizeWorkflow' => 'PhabricatorStorageManagementWorkflow',
'PhabricatorStorageManagementPartitionWorkflow' => 'PhabricatorStorageManagementWorkflow', 'PhabricatorStorageManagementPartitionWorkflow' => 'PhabricatorStorageManagementWorkflow',
'PhabricatorStorageManagementProbeWorkflow' => 'PhabricatorStorageManagementWorkflow', 'PhabricatorStorageManagementProbeWorkflow' => 'PhabricatorStorageManagementWorkflow',
'PhabricatorStorageManagementQuickstartWorkflow' => 'PhabricatorStorageManagementWorkflow', 'PhabricatorStorageManagementQuickstartWorkflow' => 'PhabricatorStorageManagementWorkflow',

View file

@ -0,0 +1,86 @@
<?php
final class PhabricatorStorageManagementOptimizeWorkflow
extends PhabricatorStorageManagementWorkflow {
protected function didConstruct() {
$this
->setName('optimize')
->setExamples('**optimize**')
->setSynopsis(pht('Run "OPTIMIZE TABLE" on tables to reclaim space.'));
}
public function didExecute(PhutilArgumentParser $args) {
$api = $this->getSingleAPI();
$conn = $api->getConn(null);
$patches = $this->getPatches();
$databases = $api->getDatabaseList($patches, true);
$total_bytes = 0;
foreach ($databases as $database) {
queryfx($conn, 'USE %C', $database);
$tables = queryfx_all($conn, 'SHOW TABLE STATUS');
foreach ($tables as $table) {
$table_name = $table['Name'];
$old_bytes =
$table['Data_length'] +
$table['Index_length'] +
$table['Data_free'];
$this->logInfo(
pht('OPTIMIZE'),
pht(
'Optimizing table "%s"."%s"...',
$database,
$table_name));
$t_start = microtime(true);
queryfx(
$conn,
'OPTIMIZE TABLE %T',
$table_name);
$t_end = microtime(true);
$status = queryfx_one(
$conn,
'SHOW TABLE STATUS LIKE %s',
$table_name);
$new_bytes =
$status['Data_length'] +
$status['Index_length'] +
$status['Data_free'];
$duration_ms = (int)(1000 * ($t_end - $t_start));
if ($old_bytes > $new_bytes) {
$this->logOkay(
pht('DONE'),
pht(
'Compacted table by %s in %sms.',
phutil_format_bytes($old_bytes - $new_bytes),
new PhutilNumber($duration_ms)));
} else {
$this->logInfo(
pht('DONE'),
pht(
'Optimized table (in %sms) but it had little effect.',
new PhutilNumber($duration_ms)));
}
$total_bytes += ($old_bytes - $new_bytes);
}
}
$this->logOkay(
pht('OPTIMIZED'),
pht(
'Completed optimizations, reclaimed %s of disk space.',
phutil_format_bytes($total_bytes)));
return 0;
}
}