When showing a small piece of a Harbormaster build log, load a small piece of data instead of the entire log

Summary: Depends on D19148. Ref T13088. The new rendering always executes range requests for data it needs, and we can satisfy these requests by loading the smallest number of chunks which span that range.

Test Plan: Piped 50,000 lines of Apache log into Harbormaster, viewed it in the new UI, got sensible rendering times and a reasonable amount of data actually going over the wire.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19149

resources/sql/autopatches/20180228.log.01.offset.sql

@@ -0,0 +1,5 @@
+ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlogchunk
+  ADD headOffset BIGINT UNSIGNED NOT NULL;
+
+ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlogchunk
+  ADD tailOffset BIGINT UNSIGNED NOT NULL;

src/applications/harbormaster/storage/build/HarbormasterBuildLog.php

@@ -130,7 +130,85 @@ final class HarbormasterBuildLog
   }
 
   public function loadData($offset, $length) {
-    return substr($this->getLogText(), $offset, $length);
+    $end = ($offset + $length);
+
+    $chunks = id(new HarbormasterBuildLogChunk())->loadAllWhere(
+      'logID = %d AND headOffset < %d AND tailOffset >= %d
+        ORDER BY headOffset ASC',
+      $this->getID(),
+      $end,
+      $offset);
+
+    // Make sure that whatever we read out of the database is a single
+    // contiguous range which contains all of the requested bytes.
+    $ranges = array();
+    foreach ($chunks as $chunk) {
+      $ranges[] = array(
+        'head' => $chunk->getHeadOffset(),
+        'tail' => $chunk->getTailOffset(),
+      );
+    }
+
+    $ranges = isort($ranges, 'head');
+    $ranges = array_values($ranges);
+    $count = count($ranges);
+    for ($ii = 0; $ii < ($count - 1); $ii++) {
+      if ($ranges[$ii + 1]['head'] === $ranges[$ii]['tail']) {
+        $ranges[$ii + 1]['head'] = $ranges[$ii]['head'];
+        unset($ranges[$ii]);
+      }
+    }
+
+    if (count($ranges) !== 1) {
+      $display_ranges = array();
+      foreach ($ranges as $range) {
+        $display_ranges[] = pht(
+          '(%d - %d)',
+          $range['head'],
+          $range['tail']);
+      }
+
+      if (!$display_ranges) {
+        $display_ranges[] = pht('<null>');
+      }
+
+      throw new Exception(
+        pht(
+          'Attempt to load log bytes (%d - %d) failed: failed to '.
+          'load a single contiguous range. Actual ranges: %s.',
+          implode('; ', $display_ranges)));
+    }
+
+    $range = head($ranges);
+    if ($range['head'] > $offset || $range['tail'] < $end) {
+      throw new Exception(
+        pht(
+          'Attempt to load log bytes (%d - %d) failed: the loaded range '.
+          '(%d - %d) does not span the requested range.',
+          $offset,
+          $end,
+          $range['head'],
+          $range['tail']));
+    }
+
+    $parts = array();
+    foreach ($chunks as $chunk) {
+      $parts[] = $chunk->getChunkDisplayText();
+    }
+    $parts = implode('', $parts);
+
+    $chop_head = ($offset - $range['head']);
+    $chop_tail = ($range['tail'] - $end);
+
+    if ($chop_head) {
+      $parts = substr($parts, $chop_head);
+    }
+
+    if ($chop_tail) {
+      $parts = substr($parts, 0, -$chop_tail);
+    }
+
+    return $parts;
   }
 
   public function getReadPosition($read_offset) {
@@ -220,17 +298,18 @@ final class HarbormasterBuildLog
 
     $this->openTransaction();
 
+    $offset = 0;
     foreach ($chunks as $chunk) {
       $rope->append($chunk->getChunkDisplayText());
       $chunk->delete();
 
       while ($rope->getByteLength() > $byte_limit) {
-        $this->writeEncodedChunk($rope, $byte_limit, $mode);
+        $offset += $this->writeEncodedChunk($rope, $offset, $byte_limit, $mode);
       }
     }
 
     while ($rope->getByteLength()) {
-      $this->writeEncodedChunk($rope, $byte_limit, $mode);
+      $offset += $this->writeEncodedChunk($rope, $offset, $byte_limit, $mode);
     }
 
     $this
@@ -240,7 +319,12 @@ final class HarbormasterBuildLog
     $this->saveTransaction();
   }
 
-  private function writeEncodedChunk(PhutilRope $rope, $length, $mode) {
+  private function writeEncodedChunk(
+    PhutilRope $rope,
+    $offset,
+    $length,
+    $mode) {
+
     $data = $rope->getPrefixBytes($length);
     $size = strlen($data);
 
@@ -258,15 +342,22 @@ final class HarbormasterBuildLog
         throw new Exception(pht('Unknown chunk encoding "%s"!', $mode));
     }
 
-    $this->writeChunk($mode, $size, $data);
+    $this->writeChunk($mode, $offset, $size, $data);
 
     $rope->removeBytesFromHead($size);
+
+    return $size;
   }
 
-  private function writeChunk($encoding, $raw_size, $data) {
+  private function writeChunk($encoding, $offset, $raw_size, $data) {
+    $head_offset = $offset;
+    $tail_offset = $offset + $raw_size;
+
     return id(new HarbormasterBuildLogChunk())
       ->setLogID($this->getID())
       ->setEncoding($encoding)
+      ->setHeadOffset($head_offset)
+      ->setTailOffset($tail_offset)
       ->setSize($raw_size)
       ->setChunk($data)
       ->save();
@@ -397,13 +488,23 @@ final class HarbormasterBuildLog
         if ($append_id) {
           queryfx(
             $conn_w,
-            'UPDATE %T SET chunk = CONCAT(chunk, %B), size = %d WHERE id = %d',
+            'UPDATE %T SET
+                chunk = CONCAT(chunk, %B),
+                size = %d,
+                tailOffset = headOffset + %d,
+              WHERE
+                id = %d',
             $chunk_table,
             $append_data,
             $prefix_size + $data_size,
+            $prefix_size + $data_size,
             $append_id);
         } else {
-          $this->writeChunk($encoding_text, $data_size, $append_data);
+          $this->writeChunk(
+            $encoding_text,
+            $this->getByteLength(),
+            $data_size,
+            $append_data);
         }
 
         $this->updateLineMap($append_data);

src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php

@@ -5,6 +5,8 @@ final class HarbormasterBuildLogChunk
 
   protected $logID;
   protected $encoding;
+  protected $headOffset;
+  protected $tailOffset;
   protected $size;
   protected $chunk;
 
@@ -20,6 +22,8 @@ final class HarbormasterBuildLogChunk
       self::CONFIG_COLUMN_SCHEMA => array(
         'logID' => 'id',
         'encoding' => 'text32',
+        'headOffset' => 'uint64',
+        'tailOffset' => 'uint64',
 
         // T6203/NULLABILITY
         // Both the type and nullability of this column are crazily wrong.
@@ -28,8 +32,8 @@ final class HarbormasterBuildLogChunk
         'chunk' => 'bytes',
       ),
       self::CONFIG_KEY_SCHEMA => array(
-        'key_log' => array(
-          'columns' => array('logID'),
+        'key_offset' => array(
+          'columns' => array('logID', 'headOffset', 'tailOffset'),
         ),
       ),
     ) + parent::getConfiguration();