From 8a2604cf06a8f3d2178c1311b8aa25917ad27a1d Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 22 Feb 2018 18:22:59 -0800 Subject: [PATCH] Add a "filePHID" to HarbormasterBuildLog and copy logs into Files during finalization Summary: Depends on D19131. Ref T13088. During log finalization, stream the log into Files to support "Download Log", archive to Files, and API access. Test Plan: Ran `write-log` and `rebuild-log`, saw Files objects generate with log content and appropriate permissions. Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam Maniphest Tasks: T13088 Differential Revision: https://secure.phabricator.com/D19132 --- .../autopatches/20180222.log.01.filephid.sql | 2 + src/__phutil_library_map__.php | 2 + ...rbormasterManagementRebuildLogWorkflow.php | 54 ++++++++++++++ ...HarbormasterManagementWriteLogWorkflow.php | 8 ++- .../phid/HarbormasterBuildLogPHIDType.php | 2 + .../storage/build/HarbormasterBuildLog.php | 26 ++++--- .../HarbormasterBuildLogChunkIterator.php | 12 +++- .../worker/HarbormasterLogWorker.php | 70 ++++++++++++++++--- 8 files changed, 153 insertions(+), 23 deletions(-) create mode 100644 resources/sql/autopatches/20180222.log.01.filephid.sql create mode 100644 src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php diff --git a/resources/sql/autopatches/20180222.log.01.filephid.sql b/resources/sql/autopatches/20180222.log.01.filephid.sql new file mode 100644 index 0000000000..a7ef2f2b3e --- /dev/null +++ b/resources/sql/autopatches/20180222.log.01.filephid.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlog + ADD filePHID VARBINARY(64); diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index d43f386009..f22f7f6730 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1309,6 +1309,7 @@ phutil_register_library_map(array( 'HarbormasterLogWorker' => 'applications/harbormaster/worker/HarbormasterLogWorker.php', 'HarbormasterManagementArchiveLogsWorkflow' => 'applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php', 'HarbormasterManagementBuildWorkflow' => 'applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php', + 'HarbormasterManagementRebuildLogWorkflow' => 'applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php', 'HarbormasterManagementRestartWorkflow' => 'applications/harbormaster/management/HarbormasterManagementRestartWorkflow.php', 'HarbormasterManagementUpdateWorkflow' => 'applications/harbormaster/management/HarbormasterManagementUpdateWorkflow.php', 'HarbormasterManagementWorkflow' => 'applications/harbormaster/management/HarbormasterManagementWorkflow.php', @@ -6614,6 +6615,7 @@ phutil_register_library_map(array( 'HarbormasterLogWorker' => 'HarbormasterWorker', 'HarbormasterManagementArchiveLogsWorkflow' => 'HarbormasterManagementWorkflow', 'HarbormasterManagementBuildWorkflow' => 'HarbormasterManagementWorkflow', + 'HarbormasterManagementRebuildLogWorkflow' => 'HarbormasterManagementWorkflow', 'HarbormasterManagementRestartWorkflow' => 'HarbormasterManagementWorkflow', 'HarbormasterManagementUpdateWorkflow' => 'HarbormasterManagementWorkflow', 'HarbormasterManagementWorkflow' => 'PhabricatorManagementWorkflow', diff --git a/src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php b/src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php new file mode 100644 index 0000000000..8644bb4515 --- /dev/null +++ b/src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php @@ -0,0 +1,54 @@ +setName('rebuild-log') + ->setExamples('**rebuild-log** --id __id__ [__options__]') + ->setSynopsis( + pht( + 'Rebuild the file and summary for a log. This is primarily '. + 'intended to make it easier to develop new log summarizers.')) + ->setArguments( + array( + array( + 'name' => 'id', + 'param' => 'id', + 'help' => pht('Log to rebuild.'), + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $viewer = $this->getViewer(); + + $log_id = $args->getArg('id'); + if (!$log_id) { + throw new PhutilArgumentUsageException( + pht('Choose a build log to rebuild with "--id".')); + } + + $log = id(new HarbormasterBuildLogQuery()) + ->setViewer($viewer) + ->withIDs(array($log_id)) + ->executeOne(); + if (!$log) { + throw new PhutilArgumentUsageException( + pht( + 'Unable to load build log "%s".', + $log_id)); + } + + PhabricatorWorker::setRunAllTasksInProcess(true); + $log->scheduleRebuild(true); + + echo tsprintf( + "%s\n", + pht('Done.')); + + return 0; + } + +} diff --git a/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php b/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php index b20204670b..367fe4fc78 100644 --- a/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php +++ b/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php @@ -7,14 +7,16 @@ final class HarbormasterManagementWriteLogWorkflow $this ->setName('write-log') ->setExamples('**write-log** --target __id__ [__options__]') - ->setSynopsis(pht('Write a new Harbormaster build log.')) + ->setSynopsis( + pht( + 'Write a new Harbormaster build log. This is primarily intended '. + 'to make development and testing easier.')) ->setArguments( array( array( 'name' => 'target', 'param' => 'id', - 'help' => pht( - 'Build Target ID to attach the log to.'), + 'help' => pht('Build Target ID to attach the log to.'), ), )); } diff --git a/src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php b/src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php index c0fba81c43..dd110556b1 100644 --- a/src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php +++ b/src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php @@ -31,6 +31,8 @@ final class HarbormasterBuildLogPHIDType extends PhabricatorPHIDType { foreach ($handles as $phid => $handle) { $build_log = $objects[$phid]; + + $handle->setName(pht('Build Log %d', $build_log->getID())); } } diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php index 2826f445de..ca8f0bb9d5 100644 --- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php +++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php @@ -9,12 +9,13 @@ final class HarbormasterBuildLog protected $logType; protected $duration; protected $live; + protected $filePHID; private $buildTarget = self::ATTACHABLE; private $rope; private $isOpen; - const CHUNK_BYTE_LIMIT = 102400; + const CHUNK_BYTE_LIMIT = 1048576; public function __construct() { $this->rope = new PhutilRope(); @@ -60,18 +61,22 @@ final class HarbormasterBuildLog ->setLive(0) ->save(); - PhabricatorWorker::scheduleTask( - 'HarbormasterLogWorker', - array( - 'logPHID' => $this->getPHID(), - ), - array( - 'objectPHID' => $this->getPHID(), - )); + $this->scheduleRebuild(false); return $this; } + public function scheduleRebuild($force) { + PhabricatorWorker::scheduleTask( + 'HarbormasterLogWorker', + array( + 'logPHID' => $this->getPHID(), + 'force' => $force, + ), + array( + 'objectPHID' => $this->getPHID(), + )); + } protected function getConfiguration() { return array( @@ -85,6 +90,7 @@ final class HarbormasterBuildLog 'duration' => 'uint32?', 'live' => 'bool', + 'filePHID' => 'phid?', ), self::CONFIG_KEY_SCHEMA => array( 'key_buildtarget' => array( @@ -180,7 +186,7 @@ final class HarbormasterBuildLog public function newChunkIterator() { return id(new HarbormasterBuildLogChunkIterator($this)) - ->setPageSize(32); + ->setPageSize(8); } private function loadLastChunkInfo() { diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php index 754248cc67..514ad7013c 100644 --- a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php +++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php @@ -5,6 +5,7 @@ final class HarbormasterBuildLogChunkIterator private $log; private $cursor; + private $asString; private $min = 0; private $max = PHP_INT_MAX; @@ -27,6 +28,11 @@ final class HarbormasterBuildLogChunkIterator return $this; } + public function setAsString($as_string) { + $this->asString = $as_string; + return $this; + } + protected function loadPage() { if ($this->cursor > $this->max) { return array(); @@ -43,7 +49,11 @@ final class HarbormasterBuildLogChunkIterator $this->cursor = last($results)->getID() + 1; } - return $results; + if ($this->asString) { + return mpull($results, 'getChunkDisplayText'); + } else { + return $results; + } } } diff --git a/src/applications/harbormaster/worker/HarbormasterLogWorker.php b/src/applications/harbormaster/worker/HarbormasterLogWorker.php index a55420bc2f..8eef7a6b6e 100644 --- a/src/applications/harbormaster/worker/HarbormasterLogWorker.php +++ b/src/applications/harbormaster/worker/HarbormasterLogWorker.php @@ -8,15 +8,6 @@ final class HarbormasterLogWorker extends HarbormasterWorker { $data = $this->getTaskData(); $log_phid = idx($data, 'logPHID'); - $log = id(new HarbormasterBuildLogQuery()) - ->setViewer($viewer) - ->withPHIDs(array($log_phid)) - ->executeOne(); - if (!$log) { - throw new PhabricatorWorkerPermanentFailureException( - pht('Invalid build log PHID "%s".', $log_phid)); - } - $phid_key = PhabricatorHash::digestToLength($log_phid, 14); $lock_key = "build.log({$phid_key})"; $lock = PhabricatorGlobalLock::newLock($lock_key); @@ -29,6 +20,25 @@ final class HarbormasterLogWorker extends HarbormasterWorker { $caught = null; try { + $log = id(new HarbormasterBuildLogQuery()) + ->setViewer($viewer) + ->withPHIDs(array($log_phid)) + ->executeOne(); + if (!$log) { + throw new PhabricatorWorkerPermanentFailureException( + pht( + 'Invalid build log PHID "%s".', + $log_phid)); + } + + if ($log->getLive()) { + throw new PhabricatorWorkerPermanentFailureException( + pht( + 'Log "%s" is still live. Logs can not be finalized until '. + 'they have closed.', + $log_phid)); + } + $this->finalizeBuildLog($log); } catch (Exception $ex) { $caught = $ex; @@ -42,9 +52,51 @@ final class HarbormasterLogWorker extends HarbormasterWorker { } private function finalizeBuildLog(HarbormasterBuildLog $log) { + $viewer = $this->getViewer(); + + $data = $this->getTaskData(); + $is_force = idx($data, 'force'); + if ($log->canCompressLog()) { $log->compressLog(); } + + if ($is_force) { + $file_phid = $log->getFilePHID(); + if ($file_phid) { + $file = id(new PhabricatorFileQuery()) + ->setViewer($viewer) + ->withPHIDs(array($file_phid)) + ->executeOne(); + if ($file) { + id(new PhabricatorDestructionEngine()) + ->destroyObject($file); + } + $log + ->setFilePHID(null) + ->save(); + } + } + + if (!$log->getFilePHID()) { + $iterator = $log->newChunkIterator() + ->setAsString(true); + + $source = id(new PhabricatorIteratorFileUploadSource()) + ->setName('harbormaster-log-'.$log->getID().'.log') + ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE) + ->setMIMEType('application/octet-stream') + ->setIterator($iterator); + + $file = $source->uploadFile(); + + $file->attachToObject($log->getPHID()); + + $log + ->setFilePHID($file->getPHID()) + ->save(); + } + } }