1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-22 14:52:41 +01:00

Store the Harbormaster log chunk format on the log record

Summary: Depends on D19137. Ref T13088. This allows `rebuild-log` to skip work if the chunks are already compressed. It also prepares for a future GC which is looking for "text" or "gzip" chunks to throw away in favor of archival into Files; such a GC can use this column to find collectable logs and then write "file" to it, meaning "chunks are gone, this data is only available in Files".

Test Plan: Ran migration, saw logs populate as "text". Ran `rebuild-log`, saw logs rebuild as "gzip".

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19138
This commit is contained in:
epriestley 2018-02-23 06:07:14 -08:00
parent 46d735d312
commit d6311044bb
4 changed files with 25 additions and 7 deletions

View file

@ -0,0 +1,2 @@
ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlog
ADD chunkFormat VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT};

View file

@ -0,0 +1,2 @@
UPDATE {$NAMESPACE}_harbormaster.harbormaster_buildlog
SET chunkFormat = 'text' WHERE chunkFormat = '';

View file

@ -13,6 +13,7 @@ final class HarbormasterBuildLog
protected $live;
protected $filePHID;
protected $byteLength;
protected $chunkFormat;
private $buildTarget = self::ATTACHABLE;
private $rope;
@ -44,7 +45,8 @@ final class HarbormasterBuildLog
->setBuildTargetPHID($build_target->getPHID())
->setDuration(null)
->setLive(1)
->setByteLength(0);
->setByteLength(0)
->setChunkFormat(HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT);
}
public function scheduleRebuild($force) {
@ -73,6 +75,7 @@ final class HarbormasterBuildLog
'live' => 'bool',
'filePHID' => 'phid?',
'byteLength' => 'uint64',
'chunkFormat' => 'text32',
),
self::CONFIG_KEY_SCHEMA => array(
'key_buildtarget' => array(
@ -105,6 +108,11 @@ final class HarbormasterBuildLog
->setPageSize(8);
}
public function newDataIterator() {
return $this->newChunkIterator()
->setAsString(true);
}
private function loadLastChunkInfo() {
$chunk_table = new HarbormasterBuildLogChunk();
$conn_w = $chunk_table->establishConnection('w');
@ -188,6 +196,10 @@ final class HarbormasterBuildLog
$this->writeEncodedChunk($rope, $byte_limit, $mode);
}
$this
->setChunkFormat($mode)
->save();
$this->saveTransaction();
}

View file

@ -58,20 +58,23 @@ final class HarbormasterLogWorker extends HarbormasterWorker {
$is_force = idx($data, 'force');
if (!$log->getByteLength() || $is_force) {
$iterator = $log->newChunkIterator()
->setAsString(true);
$iterator = $log->newDataIterator();
$byte_length = 0;
foreach ($iterator as $block) {
$byte_length += strlen($block);
}
$log
->setByteLength($byte_length)
->save();
}
if ($log->canCompressLog()) {
$log->compressLog();
$format_text = HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT;
if (($log->getChunkFormat() === $format_text) || $is_force) {
if ($log->canCompressLog()) {
$log->compressLog();
}
}
if ($is_force) {
@ -79,8 +82,7 @@ final class HarbormasterLogWorker extends HarbormasterWorker {
}
if (!$log->getFilePHID()) {
$iterator = $log->newChunkIterator()
->setAsString(true);
$iterator = $log->newDataIterator();
$source = id(new PhabricatorIteratorFileUploadSource())
->setName('harbormaster-log-'.$log->getID().'.log')