From 6514237c0e014b0d1768999673a95c38fd8ccf2e Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 1 Mar 2016 12:22:14 -0800 Subject: [PATCH] Implement an iterator for build log chunks Summary: Ref T5822. This will make it easier to compress and archive chunks without needing to hold them in memory. Test Plan: Ran a build, looked at some logs. Reviewers: chad Reviewed By: chad Maniphest Tasks: T5822 Differential Revision: https://secure.phabricator.com/D15378 --- src/__phutil_library_map__.php | 2 + .../storage/build/HarbormasterBuildLog.php | 41 +++++++------------ .../build/HarbormasterBuildLogChunk.php | 23 +++++++++++ .../HarbormasterBuildLogChunkIterator.php | 35 ++++++++++++++++ 4 files changed, 75 insertions(+), 26 deletions(-) create mode 100644 src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index b16ece7018..103dc46c51 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1045,6 +1045,7 @@ phutil_register_library_map(array( 'HarbormasterBuildLintMessage' => 'applications/harbormaster/storage/build/HarbormasterBuildLintMessage.php', 'HarbormasterBuildLog' => 'applications/harbormaster/storage/build/HarbormasterBuildLog.php', 'HarbormasterBuildLogChunk' => 'applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php', + 'HarbormasterBuildLogChunkIterator' => 'applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php', 'HarbormasterBuildLogPHIDType' => 'applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php', 'HarbormasterBuildLogQuery' => 'applications/harbormaster/query/HarbormasterBuildLogQuery.php', 'HarbormasterBuildMessage' => 'applications/harbormaster/storage/HarbormasterBuildMessage.php', @@ -5193,6 +5194,7 @@ phutil_register_library_map(array( 'PhabricatorPolicyInterface', ), 'HarbormasterBuildLogChunk' => 'HarbormasterDAO', + 'HarbormasterBuildLogChunkIterator' => 'PhutilBufferedIterator', 'HarbormasterBuildLogPHIDType' => 'PhabricatorPHIDType', 'HarbormasterBuildLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery', 'HarbormasterBuildMessage' => array( diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php index d5eaac8890..da77701d2c 100644 --- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php +++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php @@ -16,11 +16,6 @@ final class HarbormasterBuildLog const CHUNK_BYTE_LIMIT = 102400; - /** - * The log is encoded as plain text. - */ - const ENCODING_TEXT = 'text'; - public function __construct() { $this->rope = new PhutilRope(); } @@ -129,6 +124,8 @@ final class HarbormasterBuildLog $chunk_table = id(new HarbormasterBuildLogChunk())->getTableName(); $chunk_limit = self::CHUNK_BYTE_LIMIT; + $encoding_text = HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT; + $rope = $this->rope; while (true) { @@ -147,7 +144,7 @@ final class HarbormasterBuildLog $can_append = ($tail) && - ($tail['encoding'] == self::ENCODING_TEXT) && + ($tail['encoding'] == $encoding_text) && ($tail['size'] < $chunk_limit); if ($can_append) { $append_id = $tail['id']; @@ -176,7 +173,7 @@ final class HarbormasterBuildLog VALUES (%d, %s, %d, %B)', $chunk_table, $this->getID(), - self::ENCODING_TEXT, + $encoding_text, $data_size, $append_data); } @@ -185,29 +182,21 @@ final class HarbormasterBuildLog } } + public function newChunkIterator() { + return new HarbormasterBuildLogChunkIterator($this); + } + public function getLogText() { - // TODO: This won't cope very well if we're pulling like a 700MB - // log file out of the DB. We should probably implement some sort - // of optional limit parameter so that when we're rendering out only - // 25 lines in the UI, we don't wastefully read in the whole log. + // TODO: Remove this method since it won't scale for big logs. - // We have to read our content out of the database and stitch all of - // the log data back together. - $conn = $this->establishConnection('r'); - $result = queryfx_all( - $conn, - 'SELECT chunk '. - 'FROM %T '. - 'WHERE logID = %d '. - 'ORDER BY id ASC', - id(new HarbormasterBuildLogChunk())->getTableName(), - $this->getID()); + $all_chunks = $this->newChunkIterator(); - $content = ''; - foreach ($result as $row) { - $content .= $row['chunk']; + $full_text = array(); + foreach ($all_chunks as $chunk) { + $full_text[] = $chunk->getChunkDisplayText(); } - return $content; + + return implode('', $full_text); } diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php index aa1c0e1185..a7438b3ac4 100644 --- a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php +++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php @@ -8,6 +8,12 @@ final class HarbormasterBuildLogChunk protected $size; protected $chunk; + + /** + * The log is encoded as plain text. + */ + const CHUNK_ENCODING_TEXT = 'text'; + protected function getConfiguration() { return array( self::CONFIG_TIMESTAMPS => false, @@ -29,4 +35,21 @@ final class HarbormasterBuildLogChunk ) + parent::getConfiguration(); } + public function getChunkDisplayText() { + $data = $this->getChunk(); + $encoding = $this->getEncoding(); + + switch ($encoding) { + case self::CHUNK_ENCODING_TEXT: + // Do nothing, data is already plaintext. + break; + default: + throw new Exception( + pht('Unknown log chunk encoding ("%s")!', $encoding)); + } + + return $data; + } + + } diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php new file mode 100644 index 0000000000..f5e86e0062 --- /dev/null +++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php @@ -0,0 +1,35 @@ +log = $log; + } + + protected function didRewind() { + $this->cursor = 0; + } + + public function key() { + return $this->current()->getID(); + } + + protected function loadPage() { + $results = id(new HarbormasterBuildLogChunk())->loadAllWhere( + 'logID = %d AND id > %d ORDER BY id ASC LIMIT %d', + $this->log->getID(), + $this->cursor, + $this->getPageSize()); + + if ($results) { + $this->cursor = last($results)->getID(); + } + + return $results; + } + +}