mirror of
https://we.phorge.it/source/phorge.git
synced 2024-11-09 16:32:39 +01:00
Store the Harbormaster log chunk format on the log record
Summary: Depends on D19137. Ref T13088. This allows `rebuild-log` to skip work if the chunks are already compressed. It also prepares for a future GC which is looking for "text" or "gzip" chunks to throw away in favor of archival into Files; such a GC can use this column to find collectable logs and then write "file" to it, meaning "chunks are gone, this data is only available in Files". Test Plan: Ran migration, saw logs populate as "text". Ran `rebuild-log`, saw logs rebuild as "gzip". Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam Maniphest Tasks: T13088 Differential Revision: https://secure.phabricator.com/D19138
This commit is contained in:
parent
46d735d312
commit
d6311044bb
4 changed files with 25 additions and 7 deletions
|
@ -0,0 +1,2 @@
|
|||
ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlog
|
||||
ADD chunkFormat VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT};
|
|
@ -0,0 +1,2 @@
|
|||
UPDATE {$NAMESPACE}_harbormaster.harbormaster_buildlog
|
||||
SET chunkFormat = 'text' WHERE chunkFormat = '';
|
|
@ -13,6 +13,7 @@ final class HarbormasterBuildLog
|
|||
protected $live;
|
||||
protected $filePHID;
|
||||
protected $byteLength;
|
||||
protected $chunkFormat;
|
||||
|
||||
private $buildTarget = self::ATTACHABLE;
|
||||
private $rope;
|
||||
|
@ -44,7 +45,8 @@ final class HarbormasterBuildLog
|
|||
->setBuildTargetPHID($build_target->getPHID())
|
||||
->setDuration(null)
|
||||
->setLive(1)
|
||||
->setByteLength(0);
|
||||
->setByteLength(0)
|
||||
->setChunkFormat(HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT);
|
||||
}
|
||||
|
||||
public function scheduleRebuild($force) {
|
||||
|
@ -73,6 +75,7 @@ final class HarbormasterBuildLog
|
|||
'live' => 'bool',
|
||||
'filePHID' => 'phid?',
|
||||
'byteLength' => 'uint64',
|
||||
'chunkFormat' => 'text32',
|
||||
),
|
||||
self::CONFIG_KEY_SCHEMA => array(
|
||||
'key_buildtarget' => array(
|
||||
|
@ -105,6 +108,11 @@ final class HarbormasterBuildLog
|
|||
->setPageSize(8);
|
||||
}
|
||||
|
||||
public function newDataIterator() {
|
||||
return $this->newChunkIterator()
|
||||
->setAsString(true);
|
||||
}
|
||||
|
||||
private function loadLastChunkInfo() {
|
||||
$chunk_table = new HarbormasterBuildLogChunk();
|
||||
$conn_w = $chunk_table->establishConnection('w');
|
||||
|
@ -188,6 +196,10 @@ final class HarbormasterBuildLog
|
|||
$this->writeEncodedChunk($rope, $byte_limit, $mode);
|
||||
}
|
||||
|
||||
$this
|
||||
->setChunkFormat($mode)
|
||||
->save();
|
||||
|
||||
$this->saveTransaction();
|
||||
}
|
||||
|
||||
|
|
|
@ -58,20 +58,23 @@ final class HarbormasterLogWorker extends HarbormasterWorker {
|
|||
$is_force = idx($data, 'force');
|
||||
|
||||
if (!$log->getByteLength() || $is_force) {
|
||||
$iterator = $log->newChunkIterator()
|
||||
->setAsString(true);
|
||||
$iterator = $log->newDataIterator();
|
||||
|
||||
$byte_length = 0;
|
||||
foreach ($iterator as $block) {
|
||||
$byte_length += strlen($block);
|
||||
}
|
||||
|
||||
$log
|
||||
->setByteLength($byte_length)
|
||||
->save();
|
||||
}
|
||||
|
||||
if ($log->canCompressLog()) {
|
||||
$log->compressLog();
|
||||
$format_text = HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT;
|
||||
if (($log->getChunkFormat() === $format_text) || $is_force) {
|
||||
if ($log->canCompressLog()) {
|
||||
$log->compressLog();
|
||||
}
|
||||
}
|
||||
|
||||
if ($is_force) {
|
||||
|
@ -79,8 +82,7 @@ final class HarbormasterLogWorker extends HarbormasterWorker {
|
|||
}
|
||||
|
||||
if (!$log->getFilePHID()) {
|
||||
$iterator = $log->newChunkIterator()
|
||||
->setAsString(true);
|
||||
$iterator = $log->newDataIterator();
|
||||
|
||||
$source = id(new PhabricatorIteratorFileUploadSource())
|
||||
->setName('harbormaster-log-'.$log->getID().'.log')
|
||||
|
|
Loading…
Reference in a new issue