Compress Harbormaster build logs inline

Summary:
Ref T5822.

  - After a log is closed, compress it if possible.
  - Provide `bin/harbormaster archive-logs` to make it easier to change the storage format of logs.

Test Plan:
  - Ran `bin/harbormaster archive-logs` on a bunch of logs, compressing and decompressing them without issues (same hashes, same decompressed size across multiple iterations).
  - Ran new builds, verified logs were compressed after they closed.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T5822

Differential Revision: https://secure.phabricator.com/D15380

src/__phutil_library_map__.php

@@ -1114,6 +1114,7 @@ phutil_register_library_map(array(
     'HarbormasterLeaseWorkingCopyBuildStepImplementation' => 'applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php',
     'HarbormasterLintMessagesController' => 'applications/harbormaster/controller/HarbormasterLintMessagesController.php',
     'HarbormasterLintPropertyView' => 'applications/harbormaster/view/HarbormasterLintPropertyView.php',
+    'HarbormasterManagementArchiveLogsWorkflow' => 'applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php',
     'HarbormasterManagementBuildWorkflow' => 'applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php',
     'HarbormasterManagementUpdateWorkflow' => 'applications/harbormaster/management/HarbormasterManagementUpdateWorkflow.php',
     'HarbormasterManagementWorkflow' => 'applications/harbormaster/management/HarbormasterManagementWorkflow.php',
@@ -5287,6 +5288,7 @@ phutil_register_library_map(array(
     'HarbormasterLeaseWorkingCopyBuildStepImplementation' => 'HarbormasterBuildStepImplementation',
     'HarbormasterLintMessagesController' => 'HarbormasterController',
     'HarbormasterLintPropertyView' => 'AphrontView',
+    'HarbormasterManagementArchiveLogsWorkflow' => 'HarbormasterManagementWorkflow',
     'HarbormasterManagementBuildWorkflow' => 'HarbormasterManagementWorkflow',
     'HarbormasterManagementUpdateWorkflow' => 'HarbormasterManagementWorkflow',
     'HarbormasterManagementWorkflow' => 'PhabricatorManagementWorkflow',

src/applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php

@@ -0,0 +1,150 @@
+<?php
+
+final class HarbormasterManagementArchiveLogsWorkflow
+  extends HarbormasterManagementWorkflow {
+
+  protected function didConstruct() {
+    $this
+      ->setName('archive-logs')
+      ->setExamples('**archive-logs** [__options__] --mode __mode__')
+      ->setSynopsis(pht('Compress, decompress, store or destroy build logs.'))
+      ->setArguments(
+        array(
+          array(
+            'name' => 'mode',
+            'param' => 'mode',
+            'help' => pht(
+              'Use "plain" to remove encoding, or "compress" to compress '.
+              'logs.'),
+          ),
+          array(
+            'name' => 'details',
+            'help' => pht(
+              'Show more details about operations as they are performed. '.
+              'Slow! But also very reassuring!'),
+          ),
+        ));
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $viewer = $this->getViewer();
+
+    $mode = $args->getArg('mode');
+    if (!$mode) {
+      throw new PhutilArgumentUsageException(
+        pht('Choose an archival mode with --mode.'));
+    }
+
+    $valid_modes = array(
+      'plain',
+      'compress',
+    );
+
+    $valid_modes = array_fuse($valid_modes);
+    if (empty($valid_modes[$mode])) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'Unknown mode "%s". Valid modes are: %s.',
+          $mode,
+          implode(', ', $valid_modes)));
+    }
+
+    $log_table = new HarbormasterBuildLog();
+    $logs = new LiskMigrationIterator($log_table);
+
+    $show_details = $args->getArg('details');
+
+    if ($show_details) {
+      $total_old = 0;
+      $total_new = 0;
+    }
+
+    foreach ($logs as $log) {
+      echo tsprintf(
+        "%s\n",
+        pht('Processing Harbormaster build log #%d...', $log->getID()));
+
+      if ($show_details) {
+        $old_stats = $this->computeDetails($log);
+      }
+
+      switch ($mode) {
+        case 'plain':
+          $log->decompressLog();
+          break;
+        case 'compress':
+          $log->compressLog();
+          break;
+      }
+
+      if ($show_details) {
+        $new_stats = $this->computeDetails($log);
+        $this->printStats($old_stats, $new_stats);
+
+        $total_old += $old_stats['bytes'];
+        $total_new += $new_stats['bytes'];
+      }
+    }
+
+    if ($show_details) {
+      echo tsprintf(
+        "%s\n",
+        pht(
+          'Done. Total byte size of affected logs: %s -> %s.',
+          new PhutilNumber($total_old),
+          new PhutilNumber($total_new)));
+    }
+
+    return 0;
+  }
+
+  private function computeDetails(HarbormasterBuildLog $log) {
+    $bytes = 0;
+    $chunks = 0;
+    $hash = hash_init('sha1');
+
+    foreach ($log->newChunkIterator() as $chunk) {
+      $bytes += strlen($chunk->getChunk());
+      $chunks++;
+      hash_update($hash, $chunk->getChunkDisplayText());
+    }
+
+    return array(
+      'bytes' => $bytes,
+      'chunks' => $chunks,
+      'hash' => hash_final($hash),
+    );
+  }
+
+  private function printStats(array $old_stats, array $new_stats) {
+    echo tsprintf(
+      "    %s\n",
+      pht(
+        '%s: %s -> %s',
+        pht('Stored Bytes'),
+        new PhutilNumber($old_stats['bytes']),
+        new PhutilNumber($new_stats['bytes'])));
+
+    echo tsprintf(
+      "    %s\n",
+      pht(
+        '%s: %s -> %s',
+        pht('Stored Chunks'),
+        new PhutilNumber($old_stats['chunks']),
+        new PhutilNumber($new_stats['chunks'])));
+
+    echo tsprintf(
+      "    %s\n",
+      pht(
+        '%s: %s -> %s',
+        pht('Data Hash'),
+        $old_stats['hash'],
+        $new_stats['hash']));
+
+    if ($old_stats['hash'] !== $new_stats['hash']) {
+      throw new Exception(
+        pht('Log data hashes differ! Something is tragically wrong!'));
+    }
+  }
+
+}

src/applications/harbormaster/storage/build/HarbormasterBuildLog.php

@@ -52,9 +52,9 @@ final class HarbormasterBuildLog
       throw new Exception(pht('This build log is not open!'));
     }
 
-    // TODO: Encode the log contents in a gzipped format.
-
-    $this->reload();
+    if ($this->canCompressLog()) {
+      $this->compressLog();
+    }
 
     $start = $this->getDateCreated();
     $now = PhabricatorTime::getNow();
@@ -135,20 +135,15 @@ final class HarbormasterBuildLog
       }
 
       $conn_w = $this->establishConnection('w');
-      $tail = queryfx_one(
-        $conn_w,
-        'SELECT id, size, encoding FROM %T WHERE logID = %d
-          ORDER BY id DESC LIMIT 1',
-        $chunk_table,
-        $this->getID());
+      $last = $this->loadLastChunkInfo();
 
       $can_append =
-        ($tail) &&
-        ($tail['encoding'] == $encoding_text) &&
-        ($tail['size'] < $chunk_limit);
+        ($last) &&
+        ($last['encoding'] == $encoding_text) &&
+        ($last['size'] < $chunk_limit);
       if ($can_append) {
-        $append_id = $tail['id'];
-        $prefix_size = $tail['size'];
+        $append_id = $last['id'];
+        $prefix_size = $last['size'];
       } else {
         $append_id = null;
         $prefix_size = 0;
@@ -167,23 +162,28 @@ final class HarbormasterBuildLog
           $prefix_size + $data_size,
           $append_id);
       } else {
-        queryfx(
-          $conn_w,
-          'INSERT INTO %T (logID, encoding, size, chunk)
-            VALUES (%d, %s, %d, %B)',
-          $chunk_table,
-          $this->getID(),
-          $encoding_text,
-          $data_size,
-          $append_data);
+        $this->writeChunk($encoding_text, $data_size, $append_data);
       }
 
-      $rope->removeBytesFromHead(strlen($append_data));
+      $rope->removeBytesFromHead($data_size);
     }
   }
 
   public function newChunkIterator() {
-    return new HarbormasterBuildLogChunkIterator($this);
+    return id(new HarbormasterBuildLogChunkIterator($this))
+      ->setPageSize(32);
+  }
+
+  private function loadLastChunkInfo() {
+    $chunk_table = new HarbormasterBuildLogChunk();
+    $conn_w = $chunk_table->establishConnection('w');
+
+    return queryfx_one(
+      $conn_w,
+      'SELECT id, size, encoding FROM %T WHERE logID = %d
+        ORDER BY id DESC LIMIT 1',
+      $chunk_table->getTableName(),
+      $this->getID());
   }
 
   public function getLogText() {
@@ -199,6 +199,82 @@ final class HarbormasterBuildLog
     return implode('', $full_text);
   }
 
+  private function canCompressLog() {
+    return function_exists('gzdeflate');
+  }
+
+  public function compressLog() {
+    $this->processLog(HarbormasterBuildLogChunk::CHUNK_ENCODING_GZIP);
+  }
+
+  public function decompressLog() {
+    $this->processLog(HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT);
+  }
+
+  private function processLog($mode) {
+    $chunks = $this->newChunkIterator();
+
+    // NOTE: Because we're going to insert new chunks, we need to stop the
+    // iterator once it hits the final chunk which currently exists. Otherwise,
+    // it may start consuming chunks we just wrote and run forever.
+    $last = $this->loadLastChunkInfo();
+    if ($last) {
+      $chunks->setRange(null, $last['id']);
+    }
+
+    $byte_limit = self::CHUNK_BYTE_LIMIT;
+    $rope = new PhutilRope();
+
+    $this->openTransaction();
+
+    foreach ($chunks as $chunk) {
+      $rope->append($chunk->getChunkDisplayText());
+      $chunk->delete();
+
+      while ($rope->getByteLength() > $byte_limit) {
+        $this->writeEncodedChunk($rope, $byte_limit, $mode);
+      }
+    }
+
+    while ($rope->getByteLength()) {
+      $this->writeEncodedChunk($rope, $byte_limit, $mode);
+    }
+
+    $this->saveTransaction();
+  }
+
+  private function writeEncodedChunk(PhutilRope $rope, $length, $mode) {
+    $data = $rope->getPrefixBytes($length);
+    $size = strlen($data);
+
+    switch ($mode) {
+      case HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT:
+        // Do nothing.
+        break;
+      case HarbormasterBuildLogChunk::CHUNK_ENCODING_GZIP:
+        $data = gzdeflate($data);
+        if ($data === false) {
+          throw new Exception(pht('Failed to gzdeflate() log data!'));
+        }
+        break;
+      default:
+        throw new Exception(pht('Unknown chunk encoding "%s"!', $mode));
+    }
+
+    $this->writeChunk($mode, $size, $data);
+
+    $rope->removeBytesFromHead($size);
+  }
+
+  private function writeChunk($encoding, $raw_size, $data) {
+    return id(new HarbormasterBuildLogChunk())
+      ->setLogID($this->getID())
+      ->setEncoding($encoding)
+      ->setSize($raw_size)
+      ->setChunk($data)
+      ->save();
+  }
+
 
 /* -(  PhabricatorPolicyInterface  )----------------------------------------- */
 

src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php

@@ -8,15 +8,15 @@ final class HarbormasterBuildLogChunk
   protected $size;
   protected $chunk;
 
-
-  /**
-   * The log is encoded as plain text.
-   */
   const CHUNK_ENCODING_TEXT = 'text';
+  const CHUNK_ENCODING_GZIP = 'gzip';
 
   protected function getConfiguration() {
     return array(
       self::CONFIG_TIMESTAMPS => false,
+      self::CONFIG_BINARY => array(
+        'chunk' => true,
+      ),
       self::CONFIG_COLUMN_SCHEMA => array(
         'logID' => 'id',
         'encoding' => 'text32',
@@ -43,6 +43,12 @@ final class HarbormasterBuildLogChunk
       case self::CHUNK_ENCODING_TEXT:
         // Do nothing, data is already plaintext.
         break;
+      case self::CHUNK_ENCODING_GZIP:
+        $data = gzinflate($data);
+        if ($data === false) {
+          throw new Exception(pht('Unable to inflate log chunk!'));
+        }
+        break;
       default:
         throw new Exception(
           pht('Unknown log chunk encoding ("%s")!', $encoding));

src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php

@@ -6,27 +6,41 @@ final class HarbormasterBuildLogChunkIterator
   private $log;
   private $cursor;
 
+  private $min = 0;
+  private $max = PHP_INT_MAX;
+
   public function __construct(HarbormasterBuildLog $log) {
     $this->log = $log;
   }
 
   protected function didRewind() {
-    $this->cursor = 0;
+    $this->cursor = $this->min;
   }
 
   public function key() {
     return $this->current()->getID();
   }
 
+  public function setRange($min, $max) {
+    $this->min = (int)$min;
+    $this->max = (int)$max;
+    return $this;
+  }
+
   protected function loadPage() {
+    if ($this->cursor > $this->max) {
+      return array();
+    }
+
     $results = id(new HarbormasterBuildLogChunk())->loadAllWhere(
-      'logID = %d AND id > %d ORDER BY id ASC LIMIT %d',
+      'logID = %d AND id >= %d AND id <= %d ORDER BY id ASC LIMIT %d',
       $this->log->getID(),
       $this->cursor,
+      $this->max,
       $this->getPageSize());
 
     if ($results) {
-      $this->cursor = last($results)->getID();
+      $this->cursor = last($results)->getID() + 1;
     }
 
     return $results;