From d1421bc3a1ee464b30d826c13d3ccb763394a8df Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Sat, 8 Apr 2017 10:23:30 -0700
Subject: [PATCH] Add "bin/storage optimize" to run OPTIMIZE TABLE on
 everything

Summary:
Even with `innodb_file_per_table` enabled, individual table files on disk don't normally shrink.

For most tables, like `maniphest_task`, this is fine, since the data in the table normally never shrinks, or only shinks a tiny amount.

However, some tables (like the "worker" and "daemon" tables) grow very large during a huge import but most of the data is later deleted by garbage collection. In these cases, this lost space can be reclaimed by running `OPTIMIZE TABLE` on the tables.

Add a script to `OPTIMIZE TABLE` every table.

My primary goal here is just to reduce storage pressure on `db001` since there are a couple of "import the linux kernel" installs on that host wasting a bunch of space. We're not in any trouble, but this should buy us a good chunk of headroom.

Test Plan: Ran `bin/storage optimize` locally and manually ran `OPTIMIZE TABLE` in production, saw tables get optimized.

Reviewers: chad

Reviewed By: chad

Subscribers: cspeckmim

Differential Revision: https://secure.phabricator.com/D17640
---
 src/__phutil_library_map__.php                |  2 +
 ...catorStorageManagementOptimizeWorkflow.php | 86 +++++++++++++++++++
 2 files changed, 88 insertions(+)
 create mode 100644 src/infrastructure/storage/management/workflow/PhabricatorStorageManagementOptimizeWorkflow.php

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
index b567e5a9e9..279e9d6c44 100644
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -3940,6 +3940,7 @@ phutil_register_library_map(array(
     'PhabricatorStorageManagementDatabasesWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDatabasesWorkflow.php',
     'PhabricatorStorageManagementDestroyWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php',
     'PhabricatorStorageManagementDumpWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php',
+    'PhabricatorStorageManagementOptimizeWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementOptimizeWorkflow.php',
     'PhabricatorStorageManagementPartitionWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementPartitionWorkflow.php',
     'PhabricatorStorageManagementProbeWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementProbeWorkflow.php',
     'PhabricatorStorageManagementQuickstartWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php',
@@ -9286,6 +9287,7 @@ phutil_register_library_map(array(
     'PhabricatorStorageManagementDatabasesWorkflow' => 'PhabricatorStorageManagementWorkflow',
     'PhabricatorStorageManagementDestroyWorkflow' => 'PhabricatorStorageManagementWorkflow',
     'PhabricatorStorageManagementDumpWorkflow' => 'PhabricatorStorageManagementWorkflow',
+    'PhabricatorStorageManagementOptimizeWorkflow' => 'PhabricatorStorageManagementWorkflow',
     'PhabricatorStorageManagementPartitionWorkflow' => 'PhabricatorStorageManagementWorkflow',
     'PhabricatorStorageManagementProbeWorkflow' => 'PhabricatorStorageManagementWorkflow',
     'PhabricatorStorageManagementQuickstartWorkflow' => 'PhabricatorStorageManagementWorkflow',
diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementOptimizeWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementOptimizeWorkflow.php
new file mode 100644
index 0000000000..5c1c49b0b2
--- /dev/null
+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementOptimizeWorkflow.php
@@ -0,0 +1,86 @@
+<?php
+
+final class PhabricatorStorageManagementOptimizeWorkflow
+  extends PhabricatorStorageManagementWorkflow {
+
+  protected function didConstruct() {
+    $this
+      ->setName('optimize')
+      ->setExamples('**optimize**')
+      ->setSynopsis(pht('Run "OPTIMIZE TABLE" on tables to reclaim space.'));
+  }
+
+  public function didExecute(PhutilArgumentParser $args) {
+    $api = $this->getSingleAPI();
+    $conn = $api->getConn(null);
+
+    $patches = $this->getPatches();
+    $databases = $api->getDatabaseList($patches, true);
+
+    $total_bytes = 0;
+    foreach ($databases as $database) {
+      queryfx($conn, 'USE %C', $database);
+
+      $tables = queryfx_all($conn, 'SHOW TABLE STATUS');
+      foreach ($tables as $table) {
+        $table_name = $table['Name'];
+        $old_bytes =
+          $table['Data_length'] +
+          $table['Index_length'] +
+          $table['Data_free'];
+
+        $this->logInfo(
+          pht('OPTIMIZE'),
+          pht(
+            'Optimizing table "%s"."%s"...',
+            $database,
+            $table_name));
+
+        $t_start = microtime(true);
+        queryfx(
+          $conn,
+          'OPTIMIZE TABLE %T',
+          $table_name);
+        $t_end = microtime(true);
+
+        $status = queryfx_one(
+          $conn,
+          'SHOW TABLE STATUS LIKE %s',
+          $table_name);
+
+        $new_bytes =
+          $status['Data_length'] +
+          $status['Index_length'] +
+          $status['Data_free'];
+
+        $duration_ms = (int)(1000 * ($t_end - $t_start));
+
+        if ($old_bytes > $new_bytes) {
+          $this->logOkay(
+            pht('DONE'),
+            pht(
+              'Compacted table by %s in %sms.',
+              phutil_format_bytes($old_bytes - $new_bytes),
+              new PhutilNumber($duration_ms)));
+        } else {
+          $this->logInfo(
+            pht('DONE'),
+            pht(
+              'Optimized table (in %sms) but it had little effect.',
+              new PhutilNumber($duration_ms)));
+        }
+
+        $total_bytes += ($old_bytes - $new_bytes);
+      }
+    }
+
+    $this->logOkay(
+      pht('OPTIMIZED'),
+      pht(
+        'Completed optimizations, reclaimed %s of disk space.',
+        phutil_format_bytes($total_bytes)));
+
+    return 0;
+  }
+
+}