Add a "filePHID" to HarbormasterBuildLog and copy logs into Files during finalization

Summary: Depends on D19131. Ref T13088. During log finalization, stream the log into Files to support "Download Log", archive to Files, and API access.

Test Plan: Ran `write-log` and `rebuild-log`, saw Files objects generate with log content and appropriate permissions.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19132

resources/sql/autopatches/20180222.log.01.filephid.sql

@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlog
+  ADD filePHID VARBINARY(64);

src/__phutil_library_map__.php

@@ -1309,6 +1309,7 @@ phutil_register_library_map(array(
     'HarbormasterLogWorker' => 'applications/harbormaster/worker/HarbormasterLogWorker.php',
     'HarbormasterManagementArchiveLogsWorkflow' => 'applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php',
     'HarbormasterManagementBuildWorkflow' => 'applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php',
+    'HarbormasterManagementRebuildLogWorkflow' => 'applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php',
     'HarbormasterManagementRestartWorkflow' => 'applications/harbormaster/management/HarbormasterManagementRestartWorkflow.php',
     'HarbormasterManagementUpdateWorkflow' => 'applications/harbormaster/management/HarbormasterManagementUpdateWorkflow.php',
     'HarbormasterManagementWorkflow' => 'applications/harbormaster/management/HarbormasterManagementWorkflow.php',
@@ -6614,6 +6615,7 @@ phutil_register_library_map(array(
     'HarbormasterLogWorker' => 'HarbormasterWorker',
     'HarbormasterManagementArchiveLogsWorkflow' => 'HarbormasterManagementWorkflow',
     'HarbormasterManagementBuildWorkflow' => 'HarbormasterManagementWorkflow',
+    'HarbormasterManagementRebuildLogWorkflow' => 'HarbormasterManagementWorkflow',
     'HarbormasterManagementRestartWorkflow' => 'HarbormasterManagementWorkflow',
     'HarbormasterManagementUpdateWorkflow' => 'HarbormasterManagementWorkflow',
     'HarbormasterManagementWorkflow' => 'PhabricatorManagementWorkflow',

src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php

@@ -0,0 +1,54 @@
+<?php
+
+final class HarbormasterManagementRebuildLogWorkflow
+  extends HarbormasterManagementWorkflow {
+
+  protected function didConstruct() {
+    $this
+      ->setName('rebuild-log')
+      ->setExamples('**rebuild-log** --id __id__ [__options__]')
+      ->setSynopsis(
+        pht(
+          'Rebuild the file and summary for a log. This is primarily '.
+          'intended to make it easier to develop new log summarizers.'))
+      ->setArguments(
+        array(
+          array(
+            'name' => 'id',
+            'param' => 'id',
+            'help' => pht('Log to rebuild.'),
+          ),
+        ));
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $viewer = $this->getViewer();
+
+    $log_id = $args->getArg('id');
+    if (!$log_id) {
+      throw new PhutilArgumentUsageException(
+        pht('Choose a build log to rebuild with "--id".'));
+    }
+
+    $log = id(new HarbormasterBuildLogQuery())
+      ->setViewer($viewer)
+      ->withIDs(array($log_id))
+      ->executeOne();
+    if (!$log) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'Unable to load build log "%s".',
+          $log_id));
+    }
+
+    PhabricatorWorker::setRunAllTasksInProcess(true);
+    $log->scheduleRebuild(true);
+
+    echo tsprintf(
+      "%s\n",
+      pht('Done.'));
+
+    return 0;
+  }
+
+}

src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php

@@ -7,14 +7,16 @@ final class HarbormasterManagementWriteLogWorkflow
     $this
       ->setName('write-log')
       ->setExamples('**write-log** --target __id__ [__options__]')
-      ->setSynopsis(pht('Write a new Harbormaster build log.'))
+      ->setSynopsis(
+        pht(
+          'Write a new Harbormaster build log. This is primarily intended '.
+          'to make development and testing easier.'))
       ->setArguments(
         array(
           array(
             'name' => 'target',
             'param' => 'id',
-            'help' => pht(
-              'Build Target ID to attach the log to.'),
+            'help' => pht('Build Target ID to attach the log to.'),
           ),
         ));
   }

src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php

@@ -31,6 +31,8 @@ final class HarbormasterBuildLogPHIDType extends PhabricatorPHIDType {
 
     foreach ($handles as $phid => $handle) {
       $build_log = $objects[$phid];
+
+      $handle->setName(pht('Build Log %d', $build_log->getID()));
     }
   }
 

src/applications/harbormaster/storage/build/HarbormasterBuildLog.php

@@ -9,12 +9,13 @@ final class HarbormasterBuildLog
   protected $logType;
   protected $duration;
   protected $live;
+  protected $filePHID;
 
   private $buildTarget = self::ATTACHABLE;
   private $rope;
   private $isOpen;
 
-  const CHUNK_BYTE_LIMIT = 102400;
+  const CHUNK_BYTE_LIMIT = 1048576;
 
   public function __construct() {
     $this->rope = new PhutilRope();
@@ -60,18 +61,22 @@ final class HarbormasterBuildLog
       ->setLive(0)
       ->save();
 
-    PhabricatorWorker::scheduleTask(
-      'HarbormasterLogWorker',
-      array(
-        'logPHID' => $this->getPHID(),
-      ),
-      array(
-        'objectPHID' => $this->getPHID(),
-      ));
+    $this->scheduleRebuild(false);
 
     return $this;
   }
 
+  public function scheduleRebuild($force) {
+    PhabricatorWorker::scheduleTask(
+      'HarbormasterLogWorker',
+      array(
+        'logPHID' => $this->getPHID(),
+        'force' => $force,
+      ),
+      array(
+        'objectPHID' => $this->getPHID(),
+      ));
+  }
 
   protected function getConfiguration() {
     return array(
@@ -85,6 +90,7 @@ final class HarbormasterBuildLog
         'duration' => 'uint32?',
 
         'live' => 'bool',
+        'filePHID' => 'phid?',
       ),
       self::CONFIG_KEY_SCHEMA => array(
         'key_buildtarget' => array(
@@ -180,7 +186,7 @@ final class HarbormasterBuildLog
 
   public function newChunkIterator() {
     return id(new HarbormasterBuildLogChunkIterator($this))
-      ->setPageSize(32);
+      ->setPageSize(8);
   }
 
   private function loadLastChunkInfo() {

src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php

@@ -5,6 +5,7 @@ final class HarbormasterBuildLogChunkIterator
 
   private $log;
   private $cursor;
+  private $asString;
 
   private $min = 0;
   private $max = PHP_INT_MAX;
@@ -27,6 +28,11 @@ final class HarbormasterBuildLogChunkIterator
     return $this;
   }
 
+  public function setAsString($as_string) {
+    $this->asString = $as_string;
+    return $this;
+  }
+
   protected function loadPage() {
     if ($this->cursor > $this->max) {
       return array();
@@ -43,7 +49,11 @@ final class HarbormasterBuildLogChunkIterator
       $this->cursor = last($results)->getID() + 1;
     }
 
-    return $results;
+    if ($this->asString) {
+      return mpull($results, 'getChunkDisplayText');
+    } else {
+      return $results;
+    }
   }
 
 }

src/applications/harbormaster/worker/HarbormasterLogWorker.php

@@ -8,15 +8,6 @@ final class HarbormasterLogWorker extends HarbormasterWorker {
     $data = $this->getTaskData();
     $log_phid = idx($data, 'logPHID');
 
-    $log = id(new HarbormasterBuildLogQuery())
-      ->setViewer($viewer)
-      ->withPHIDs(array($log_phid))
-      ->executeOne();
-    if (!$log) {
-      throw new PhabricatorWorkerPermanentFailureException(
-        pht('Invalid build log PHID "%s".', $log_phid));
-    }
-
     $phid_key = PhabricatorHash::digestToLength($log_phid, 14);
     $lock_key = "build.log({$phid_key})";
     $lock = PhabricatorGlobalLock::newLock($lock_key);
@@ -29,6 +20,25 @@ final class HarbormasterLogWorker extends HarbormasterWorker {
 
     $caught = null;
     try {
+      $log = id(new HarbormasterBuildLogQuery())
+        ->setViewer($viewer)
+        ->withPHIDs(array($log_phid))
+        ->executeOne();
+      if (!$log) {
+        throw new PhabricatorWorkerPermanentFailureException(
+          pht(
+            'Invalid build log PHID "%s".',
+            $log_phid));
+      }
+
+      if ($log->getLive()) {
+        throw new PhabricatorWorkerPermanentFailureException(
+          pht(
+            'Log "%s" is still live. Logs can not be finalized until '.
+            'they have closed.',
+            $log_phid));
+      }
+
       $this->finalizeBuildLog($log);
     } catch (Exception $ex) {
       $caught = $ex;
@@ -42,9 +52,51 @@ final class HarbormasterLogWorker extends HarbormasterWorker {
   }
 
   private function finalizeBuildLog(HarbormasterBuildLog $log) {
+    $viewer = $this->getViewer();
+
+    $data = $this->getTaskData();
+    $is_force = idx($data, 'force');
+
     if ($log->canCompressLog()) {
       $log->compressLog();
     }
+
+    if ($is_force) {
+      $file_phid = $log->getFilePHID();
+      if ($file_phid) {
+        $file = id(new PhabricatorFileQuery())
+          ->setViewer($viewer)
+          ->withPHIDs(array($file_phid))
+          ->executeOne();
+        if ($file) {
+          id(new PhabricatorDestructionEngine())
+            ->destroyObject($file);
+        }
+        $log
+          ->setFilePHID(null)
+          ->save();
+      }
+    }
+
+    if (!$log->getFilePHID()) {
+      $iterator = $log->newChunkIterator()
+        ->setAsString(true);
+
+      $source = id(new PhabricatorIteratorFileUploadSource())
+        ->setName('harbormaster-log-'.$log->getID().'.log')
+        ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE)
+        ->setMIMEType('application/octet-stream')
+        ->setIterator($iterator);
+
+      $file = $source->uploadFile();
+
+      $file->attachToObject($log->getPHID());
+
+      $log
+        ->setFilePHID($file->getPHID())
+        ->save();
+    }
+
   }
 
 }