1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-29 10:12:41 +01:00

Sort of make Harbormaster build logs page properly

Summary: Depends on D19139. Ref T13088. This doesn't actually work, but is close enough that a skilled attacker might be able to briefly deceive a small child.

Test Plan:
  - Viewed some very small logs under very controlled conditions, saw content.
  - Larger logs vaguely do something resembling working correctly.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19141
This commit is contained in:
epriestley 2018-02-22 19:35:13 -08:00
parent 6dc341be87
commit 11d1dc484b
9 changed files with 721 additions and 7 deletions

View file

@ -78,7 +78,7 @@ return array(
'rsrc/css/application/feed/feed.css' => 'ecd4ec57',
'rsrc/css/application/files/global-drag-and-drop.css' => 'b556a948',
'rsrc/css/application/flag/flag.css' => 'bba8f811',
'rsrc/css/application/harbormaster/harbormaster.css' => 'f491c9f4',
'rsrc/css/application/harbormaster/harbormaster.css' => 'fecac64f',
'rsrc/css/application/herald/herald-test.css' => 'a52e323e',
'rsrc/css/application/herald/herald.css' => 'cd8d0134',
'rsrc/css/application/maniphest/report.css' => '9b9580b7',
@ -416,6 +416,7 @@ return array(
'rsrc/js/application/drydock/drydock-live-operation-status.js' => '901935ef',
'rsrc/js/application/files/behavior-icon-composer.js' => '8499b6ab',
'rsrc/js/application/files/behavior-launch-icon-composer.js' => '48086888',
'rsrc/js/application/harbormaster/behavior-harbormaster-log.js' => '0844f3c1',
'rsrc/js/application/herald/HeraldRuleEditor.js' => 'dca75c0e',
'rsrc/js/application/herald/PathTypeahead.js' => 'f7fc67ec',
'rsrc/js/application/herald/herald-rule-editor.js' => '7ebaeed3',
@ -578,7 +579,7 @@ return array(
'font-fontawesome' => 'e838e088',
'font-lato' => 'c7ccd872',
'global-drag-and-drop-css' => 'b556a948',
'harbormaster-css' => 'f491c9f4',
'harbormaster-css' => 'fecac64f',
'herald-css' => 'cd8d0134',
'herald-rule-editor' => 'dca75c0e',
'herald-test-css' => 'a52e323e',
@ -635,6 +636,7 @@ return array(
'javelin-behavior-event-all-day' => 'b41537c9',
'javelin-behavior-fancy-datepicker' => 'ecf4e799',
'javelin-behavior-global-drag-and-drop' => '960f6a39',
'javelin-behavior-harbormaster-log' => '0844f3c1',
'javelin-behavior-herald-rule-editor' => '7ebaeed3',
'javelin-behavior-high-security-warning' => 'a464fe03',
'javelin-behavior-history-install' => '7ee2b591',
@ -960,6 +962,9 @@ return array(
'javelin-stratcom',
'javelin-workflow',
),
'0844f3c1' => array(
'javelin-behavior',
),
'08f4ccc3' => array(
'phui-oi-list-view-css',
),

View file

@ -1230,6 +1230,7 @@ phutil_register_library_map(array(
'HarbormasterBuildLogDownloadController' => 'applications/harbormaster/controller/HarbormasterBuildLogDownloadController.php',
'HarbormasterBuildLogPHIDType' => 'applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php',
'HarbormasterBuildLogQuery' => 'applications/harbormaster/query/HarbormasterBuildLogQuery.php',
'HarbormasterBuildLogRenderController' => 'applications/harbormaster/controller/HarbormasterBuildLogRenderController.php',
'HarbormasterBuildLogTestCase' => 'applications/harbormaster/__tests__/HarbormasterBuildLogTestCase.php',
'HarbormasterBuildLogView' => 'applications/harbormaster/view/HarbormasterBuildLogView.php',
'HarbormasterBuildLogViewController' => 'applications/harbormaster/controller/HarbormasterBuildLogViewController.php',
@ -6519,6 +6520,7 @@ phutil_register_library_map(array(
'HarbormasterBuildLogDownloadController' => 'HarbormasterController',
'HarbormasterBuildLogPHIDType' => 'PhabricatorPHIDType',
'HarbormasterBuildLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
'HarbormasterBuildLogRenderController' => 'HarbormasterController',
'HarbormasterBuildLogTestCase' => 'PhabricatorTestCase',
'HarbormasterBuildLogView' => 'AphrontView',
'HarbormasterBuildLogViewController' => 'HarbormasterController',

View file

@ -97,7 +97,10 @@ final class PhabricatorHarbormasterApplication extends PhabricatorApplication {
'buildkite/' => 'HarbormasterBuildkiteHookController',
),
'log/' => array(
'view/(?P<id>\d+)/' => 'HarbormasterBuildLogViewController',
'view/(?P<id>\d+)/(?:\$(?P<lines>\d+(?:-\d+)?))?'
=> 'HarbormasterBuildLogViewController',
'render/(?P<id>\d+)/(?:\$(?P<lines>\d+(?:-\d+)?))?'
=> 'HarbormasterBuildLogRenderController',
'download/(?P<id>\d+)/' => 'HarbormasterBuildLogDownloadController',
),
),

View file

@ -0,0 +1,562 @@
<?php
final class HarbormasterBuildLogRenderController
extends HarbormasterController {
public function handleRequest(AphrontRequest $request) {
$viewer = $this->getViewer();
$id = $request->getURIData('id');
$log = id(new HarbormasterBuildLogQuery())
->setViewer($viewer)
->withIDs(array($id))
->executeOne();
if (!$log) {
return new Aphront404Response();
}
$log_size = $this->getTotalByteLength($log);
$head_lines = $request->getInt('head');
if ($head_lines === null) {
$head_lines = 8;
}
$head_lines = min($head_lines, 100);
$head_lines = max($head_lines, 0);
$tail_lines = $request->getInt('tail');
if ($tail_lines === null) {
$tail_lines = 16;
}
$tail_lines = min($tail_lines, 100);
$tail_lines = max($tail_lines, 0);
$head_offset = $request->getInt('headOffset');
if ($head_offset === null) {
$head_offset = 0;
}
$tail_offset = $request->getInt('tailOffset');
if ($tail_offset === null) {
$tail_offset = $log_size;
}
// Figure out which ranges we're actually going to read. We'll read either
// one range (either just at the head, or just at the tail) or two ranges
// (one at the head and one at the tail).
// This gets a little bit tricky because: the ranges may overlap; we just
// want to do one big read if there is only a little bit of text left
// between the ranges; we may not know where the tail range ends; and we
// can only read forward from line map markers, not from any arbitrary
// position in the file.
$bytes_per_line = 140;
$body_lines = 8;
$views = array();
if ($head_lines > 0) {
$views[] = array(
'offset' => $head_offset,
'lines' => $head_lines,
'direction' => 1,
);
}
if ($tail_lines > 0) {
$views[] = array(
'offset' => $tail_offset,
'lines' => $tail_lines,
'direction' => -1,
);
}
$reads = $views;
foreach ($reads as $key => $read) {
$offset = $read['offset'];
$lines = $read['lines'];
$read_length = 0;
$read_length += ($lines * $bytes_per_line);
$read_length += ($body_lines * $bytes_per_line);
$direction = $read['direction'];
if ($direction < 0) {
$offset -= $read_length;
if ($offset < 0) {
$offset = 0;
$read_length = $log_size;
}
}
$position = $log->getReadPosition($offset);
list($position_offset, $position_line) = $position;
$read_length += ($offset - $position_offset);
$reads[$key]['fetchOffset'] = $position_offset;
$reads[$key]['fetchLength'] = $read_length;
$reads[$key]['fetchLine'] = $position_line;
}
$reads = $this->mergeOverlappingReads($reads);
foreach ($reads as $key => $read) {
$data = $log->loadData($read['fetchOffset'], $read['fetchLength']);
$offset = $read['fetchOffset'];
$line = $read['fetchLine'];
$lines = $this->getLines($data);
$line_data = array();
foreach ($lines as $line_text) {
$length = strlen($line_text);
$line_data[] = array(
'offset' => $offset,
'length' => $length,
'line' => $line,
'data' => $line_text,
);
$line += 1;
$offset += $length;
}
$reads[$key]['data'] = $data;
$reads[$key]['lines'] = $line_data;
}
foreach ($views as $view_key => $view) {
$anchor_byte = $view['offset'];
$data_key = null;
foreach ($reads as $read_key => $read) {
$s = $read['fetchOffset'];
$e = $s + $read['fetchLength'];
if (($s <= $anchor_byte) && ($e >= $anchor_byte)) {
$data_key = $read_key;
break;
}
}
if ($data_key === null) {
throw new Exception(
pht('Unable to find fetch!'));
}
$anchor_key = null;
foreach ($reads[$data_key]['lines'] as $line_key => $line) {
$s = $line['offset'];
$e = $s + $line['length'];
if (($s <= $anchor_byte) && ($e >= $anchor_byte)) {
$anchor_key = $line_key;
break;
}
}
if ($anchor_key === null) {
throw new Exception(
pht(
'Unable to find lines.'));
}
if ($direction > 0) {
$slice_offset = $line_key;
} else {
$slice_offset = max(0, $line_key - ($view['lines'] - 1));
}
$slice_length = $view['lines'];
$views[$view_key] += array(
'sliceKey' => $data_key,
'sliceOffset' => $slice_offset,
'sliceLength' => $slice_length,
);
}
foreach ($views as $view_key => $view) {
$slice_key = $view['sliceKey'];
$lines = array_slice(
$reads[$slice_key]['lines'],
$view['sliceOffset'],
$view['sliceLength']);
$data_offset = null;
$data_length = null;
foreach ($lines as $line) {
if ($data_offset === null) {
$data_offset = $line['offset'];
}
$data_length += $line['length'];
}
// If the view cursor starts in the middle of a line, we're going to
// strip part of the line.
$direction = $view['direction'];
if ($direction > 0) {
$view_offset = $view['offset'];
$view_length = $data_length;
if ($data_offset < $view_offset) {
$trim = ($view_offset - $data_offset);
$view_length -= $trim;
}
} else {
$view_offset = $data_offset;
$view_length = $data_length;
if ($data_offset + $data_length > $view['offset']) {
$view_length -= (($data_offset + $data_length) - $view['offset']);
}
}
$views[$view_key] += array(
'viewOffset' => $view_offset,
'viewLength' => $view_length,
);
}
$views = $this->mergeOverlappingViews($views);
foreach ($views as $view_key => $view) {
$slice_key = $view['sliceKey'];
$lines = array_slice(
$reads[$slice_key]['lines'],
$view['sliceOffset'],
$view['sliceLength']);
$view_offset = $view['viewOffset'];
foreach ($lines as $line_key => $line) {
$line_offset = $line['offset'];
if ($line_offset >= $view_offset) {
break;
}
$trim = ($view_offset - $line_offset);
$line_data = substr($line['data'], $trim);
if (!strlen($line_data)) {
unset($lines[$line_key]);
continue;
}
$lines[$line_key]['data'] = $line_data;
$lines[$line_key]['length'] = strlen($line_data);
$lines[$line_key]['offset'] += $trim;
break;
}
$view_end = $view['viewOffset'] + $view['viewLength'];
foreach ($lines as $line_key => $line) {
$line_end = $line['offset'] + $line['length'];
if ($line_end <= $view_end) {
break;
}
$trim = ($line_end - $view_end);
$line_data = substr($line['data'], -$trim);
if (!strlen($line_data)) {
unset($lines[$line_key]);
continue;
}
$lines[$line_key]['data'] = $line_data;
$lines[$line_key]['length'] = strlen($line_data);
}
$views[$view_key]['viewData'] = $lines;
}
$spacer = null;
$render = array();
foreach ($views as $view) {
if ($spacer) {
$spacer['tail'] = $view['viewOffset'];
$render[] = $spacer;
}
$render[] = $view;
$spacer = array(
'spacer' => true,
'head' => ($view['viewOffset'] + $view['viewLength']),
);
}
$uri = $log->getURI();
$highlight_range = $request->getURIData('lines');
$rows = array();
foreach ($render as $range) {
if (isset($range['spacer'])) {
$rows[] = phutil_tag(
'tr',
array(),
array(
phutil_tag(
'th',
array(),
null),
phutil_tag(
'td',
array(),
array(
javelin_tag(
'a',
array(
'sigil' => 'harbormaster-log-expand',
'meta' => array(
'headOffset' => $range['head'],
'tailOffset' => $range['tail'],
'head' => 4,
),
),
'Show Up ^^^^'),
'... '.($range['tail'] - $range['head']).' bytes ...',
javelin_tag(
'a',
array(
'sigil' => 'harbormaster-log-expand',
'meta' => array(
'headOffset' => $range['head'],
'tailOffset' => $range['tail'],
'tail' => 4,
),
),
'Show Down VVVV'),
)),
));
continue;
}
$lines = $range['viewData'];
foreach ($lines as $line) {
$display_line = ($line['line'] + 1);
$display_text = ($line['data']);
$display_line = phutil_tag(
'a',
array(
'href' => $uri.'$'.$display_line,
),
$display_line);
$line_cell = phutil_tag('th', array(), $display_line);
$text_cell = phutil_tag('td', array(), $display_text);
$rows[] = phutil_tag(
'tr',
array(),
array(
$line_cell,
$text_cell,
));
}
}
$table = phutil_tag(
'table',
array(
'class' => 'harbormaster-log-table PhabricatorMonospaced',
),
$rows);
// When this is a normal AJAX request, return the rendered log fragment
// in an AJAX payload.
if ($request->isAjax()) {
return id(new AphrontAjaxResponse())
->setContent(
array(
'markup' => hsprintf('%s', $table),
));
}
// If the page is being accessed as a standalone page, present a
// readable version of the fragment for debugging.
require_celerity_resource('harbormaster-css');
$header = pht('Standalone Log Fragment');
$render_view = id(new PHUIObjectBoxView())
->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
->setHeaderText($header)
->appendChild($table);
$page_view = id(new PHUITwoColumnView())
->setFooter($render_view);
$crumbs = $this->buildApplicationCrumbs()
->addTextCrumb(pht('Build Log %d', $log->getID()), $log->getURI())
->addTextCrumb(pht('Fragment'))
->setBorder(true);
return $this->newPage()
->setTitle(
array(
pht('Build Log %d', $log->getID()),
pht('Standalone Fragment'),
))
->setCrumbs($crumbs)
->appendChild($page_view);
}
private function getTotalByteLength(HarbormasterBuildLog $log) {
$total_bytes = $log->getByteLength();
if ($total_bytes) {
return (int)$total_bytes;
}
// TODO: Remove this after enough time has passed for installs to run
// log rebuilds or decide they don't care about older logs.
// Older logs don't have this data denormalized onto the log record unless
// an administrator has run `bin/harbormaster rebuild-log --all` or
// similar. Try to figure it out by summing up the size of each chunk.
// Note that the log may also be legitimately empty and have actual size
// zero.
$chunk = new HarbormasterBuildLogChunk();
$conn = $chunk->establishConnection('r');
$row = queryfx_one(
$conn,
'SELECT SUM(size) total FROM %T WHERE logID = %d',
$chunk->getTableName(),
$log->getID());
return (int)$row['total'];
}
private function getLines($data) {
$parts = preg_split("/(\r\n|\r|\n)/", $data, 0, PREG_SPLIT_DELIM_CAPTURE);
if (last($parts) === '') {
array_pop($parts);
}
$lines = array();
for ($ii = 0; $ii < count($parts); $ii += 2) {
$line = $parts[$ii];
if (isset($parts[$ii + 1])) {
$line .= $parts[$ii + 1];
}
$lines[] = $line;
}
return $lines;
}
private function mergeOverlappingReads(array $reads) {
// Find planned reads which will overlap and merge them into a single
// larger read.
$uk = array_keys($reads);
$vk = array_keys($reads);
foreach ($uk as $ukey) {
foreach ($vk as $vkey) {
// Don't merge a range into itself, even though they do technically
// overlap.
if ($ukey === $vkey) {
continue;
}
$uread = idx($reads, $ukey);
if ($uread === null) {
continue;
}
$vread = idx($reads, $vkey);
if ($vread === null) {
continue;
}
$us = $uread['fetchOffset'];
$ue = $us + $uread['fetchLength'];
$vs = $vread['fetchOffset'];
$ve = $vs + $vread['fetchLength'];
if (($vs > $ue) || ($ve < $us)) {
continue;
}
$min = min($us, $vs);
$max = max($ue, $ve);
$reads[$ukey]['fetchOffset'] = $min;
$reads[$ukey]['fetchLength'] = ($max - $min);
$reads[$ukey]['fetchLine'] = min(
$uread['fetchLine'],
$vread['fetchLine']);
unset($reads[$vkey]);
}
}
return $reads;
}
private function mergeOverlappingViews(array $views) {
$uk = array_keys($views);
$vk = array_keys($views);
$body_lines = 8;
$body_bytes = ($body_lines * 140);
foreach ($uk as $ukey) {
foreach ($vk as $vkey) {
if ($ukey === $vkey) {
continue;
}
$uview = idx($views, $ukey);
if ($uview === null) {
continue;
}
$vview = idx($views, $vkey);
if ($vview === null) {
continue;
}
// If these views don't use the same line data, don't try to
// merge them.
if ($uview['sliceKey'] != $vview['sliceKey']) {
continue;
}
// If these views are overlapping or separated by only a few bytes,
// merge them into a single view.
$us = $uview['viewOffset'];
$ue = $us + $uview['viewLength'];
$vs = $vview['viewOffset'];
$ve = $vs + $vview['viewLength'];
$uss = $uview['sliceOffset'];
$use = $uss + $uview['sliceLength'];
$vss = $vview['sliceOffset'];
$vse = $vss + $vview['sliceLength'];
if ($ue <= $vs) {
if (($ue + $body_bytes) >= $vs) {
if (($use + $body_lines) >= $vss) {
$views[$ukey] = array(
'sliceLength' => ($vse - $uss),
'viewLength' => ($ve - $us),
) + $views[$ukey];
unset($views[$vkey]);
continue;
}
}
}
}
}
return $views;
}
}

View file

@ -4,8 +4,7 @@ final class HarbormasterBuildLogViewController
extends HarbormasterController {
public function handleRequest(AphrontRequest $request) {
$request = $this->getRequest();
$viewer = $request->getUser();
$viewer = $this->getViewer();
$id = $request->getURIData('id');
@ -21,7 +20,8 @@ final class HarbormasterBuildLogViewController
$log_view = id(new HarbormasterBuildLogView())
->setViewer($viewer)
->setBuildLog($log);
->setBuildLog($log)
->setHighlightedLineRange($request->getURIData('lines'));
$crumbs = $this->buildApplicationCrumbs()
->addTextCrumb(pht('Build Logs'))

View file

@ -129,6 +129,30 @@ final class HarbormasterBuildLog
$this->getID());
}
public function loadData($offset, $length) {
return substr($this->getLogText(), $offset, $length);
}
public function getReadPosition($read_offset) {
$position = array(0, 0);
$map = $this->getLineMap();
if (!$map) {
throw new Exception(pht('No line map.'));
}
list($map) = $map;
foreach ($map as $marker) {
list($offset, $count) = $marker;
if ($offset > $read_offset) {
break;
}
$position = $marker;
}
return $position;
}
public function getLogText() {
// TODO: Remove this method since it won't scale for big logs.
@ -148,6 +172,15 @@ final class HarbormasterBuildLog
return "/harbormaster/log/view/{$id}/";
}
public function getRenderURI($lines) {
if (strlen($lines)) {
$lines = '$'.$lines;
}
$id = $this->getID();
return "/harbormaster/log/render/{$id}/{$lines}";
}
/* -( Chunks )------------------------------------------------------------- */

View file

@ -3,6 +3,7 @@
final class HarbormasterBuildLogView extends AphrontView {
private $log;
private $highlightedLineRange;
public function setBuildLog(HarbormasterBuildLog $log) {
$this->log = $log;
@ -13,6 +14,15 @@ final class HarbormasterBuildLogView extends AphrontView {
return $this->log;
}
public function setHighlightedLineRange($range) {
$this->highlightedLineRange = $range;
return $this;
}
public function getHighlightedLineRange() {
return $this->highlightedLineRange;
}
public function render() {
$viewer = $this->getViewer();
$log = $this->getBuildLog();
@ -34,10 +44,28 @@ final class HarbormasterBuildLogView extends AphrontView {
$header->addActionLink($download_button);
$content_id = celerity_generate_unique_node_id();
$content_div = javelin_tag(
'div',
array(
'id' => $content_id,
'class' => 'harbormaster-log-view-loading',
),
pht('Loading...'));
require_celerity_resource('harbormaster-css');
Javelin::initBehavior(
'harbormaster-log',
array(
'contentNodeID' => $content_id,
'renderURI' => $log->getRenderURI($this->getHighlightedLineRange()),
));
$box_view = id(new PHUIObjectBoxView())
->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
->setHeader($header)
->appendChild('...');
->appendChild($content_div);
return $box_view;
}

View file

@ -30,3 +30,41 @@
text-overflow: ellipsis;
color: {$lightgreytext};
}
.harbormaster-log-view-loading {
padding: 8px;
text-align: center;
color: {$lightgreytext};
}
.harbormaster-log-table th {
background-color: {$paste.highlight};
border-right: 1px solid {$paste.border};
-moz-user-select: -moz-none;
-khtml-user-select: none;
-webkit-user-select: none;
-ms-user-select: none;
user-select: none;
}
.harbormaster-log-table th a {
display: block;
color: {$darkbluetext};
text-align: right;
padding: 2px 6px 1px 12px;
}
.harbormaster-log-table th a:hover {
background: {$paste.border};
}
.harbormaster-log-table td {
white-space: pre-wrap;
padding: 2px 8px 1px;
width: 100%;
}
.harbormaster-log-table tr.harbormaster-log-highlighted td {
background: {$paste.highlight};
}

View file

@ -0,0 +1,43 @@
/**
* @provides javelin-behavior-harbormaster-log
* @requires javelin-behavior
*/
JX.behavior('harbormaster-log', function(config) {
var contentNode = JX.$(config.contentNodeID);
JX.DOM.listen(contentNode, 'click', 'harbormaster-log-expand', function(e) {
if (!e.isNormalClick()) {
return;
}
e.kill();
var row = e.getNode('tag:tr');
var data = e.getNodeData('harbormaster-log-expand');
var uri = new JX.URI(config.renderURI)
.addQueryParams(data);
var request = new JX.Request(uri, function(r) {
var result = JX.$H(r.markup).getNode();
var rows = JX.DOM.scry(result, 'tr');
JX.DOM.replace(row, rows);
});
request.send();
});
function onresponse(r) {
JX.DOM.alterClass(contentNode, 'harbormaster-log-view-loading', false);
JX.DOM.setContent(contentNode, JX.$H(r.markup));
}
var uri = new JX.URI(config.renderURI);
new JX.Request(uri, onresponse)
.send();
});