mirror of
https://we.phorge.it/source/phorge.git
synced 2024-11-22 14:52:41 +01:00
Add a very rough, proof-of-concept Jupyter notebook document engine
Summary: Depends on D19252. Ref T13105. This very roughly renders Jupyter notebooks. It's probably better than showing the raw JSON, but not by much. Test Plan: - Viewed various notebooks with various cell types, including markdown, code, stdout, stderr, images, HTML, and Javascript. - HTML and Javascript are not live-fired since they're wildly dangerous. Maniphest Tasks: T13105 Differential Revision: https://secure.phabricator.com/D19253
This commit is contained in:
parent
fb4ce851c4
commit
cbf3d3c371
5 changed files with 366 additions and 3 deletions
|
@ -9,7 +9,7 @@ return array(
|
|||
'names' => array(
|
||||
'conpherence.pkg.css' => 'e68cf1fa',
|
||||
'conpherence.pkg.js' => '15191c65',
|
||||
'core.pkg.css' => '2d73b2f3',
|
||||
'core.pkg.css' => '7daac340',
|
||||
'core.pkg.js' => 'b9b4a943',
|
||||
'differential.pkg.css' => '113e692c',
|
||||
'differential.pkg.js' => 'f6d809c0',
|
||||
|
@ -168,7 +168,7 @@ return array(
|
|||
'rsrc/css/phui/phui-object-box.css' => '9cff003c',
|
||||
'rsrc/css/phui/phui-pager.css' => 'edcbc226',
|
||||
'rsrc/css/phui/phui-pinboard-view.css' => '2495140e',
|
||||
'rsrc/css/phui/phui-property-list-view.css' => '47018d3c',
|
||||
'rsrc/css/phui/phui-property-list-view.css' => '871f6815',
|
||||
'rsrc/css/phui/phui-remarkup-preview.css' => '54a34863',
|
||||
'rsrc/css/phui/phui-segment-bar-view.css' => 'b1d1b892',
|
||||
'rsrc/css/phui/phui-spacing.css' => '042804d6',
|
||||
|
@ -850,7 +850,7 @@ return array(
|
|||
'phui-oi-simple-ui-css' => 'a8beebea',
|
||||
'phui-pager-css' => 'edcbc226',
|
||||
'phui-pinboard-view-css' => '2495140e',
|
||||
'phui-property-list-view-css' => '47018d3c',
|
||||
'phui-property-list-view-css' => '871f6815',
|
||||
'phui-remarkup-preview-css' => '54a34863',
|
||||
'phui-segment-bar-view-css' => 'b1d1b892',
|
||||
'phui-spacing-css' => '042804d6',
|
||||
|
|
|
@ -3190,6 +3190,7 @@ phutil_register_library_map(array(
|
|||
'PhabricatorJSONExportFormat' => 'infrastructure/export/format/PhabricatorJSONExportFormat.php',
|
||||
'PhabricatorJavelinLinter' => 'infrastructure/lint/linter/PhabricatorJavelinLinter.php',
|
||||
'PhabricatorJiraIssueHasObjectEdgeType' => 'applications/doorkeeper/edge/PhabricatorJiraIssueHasObjectEdgeType.php',
|
||||
'PhabricatorJupyterDocumentEngine' => 'applications/files/document/PhabricatorJupyterDocumentEngine.php',
|
||||
'PhabricatorKeyValueDatabaseCache' => 'applications/cache/PhabricatorKeyValueDatabaseCache.php',
|
||||
'PhabricatorKeyValueSerializingCacheProxy' => 'applications/cache/PhabricatorKeyValueSerializingCacheProxy.php',
|
||||
'PhabricatorKeyboardRemarkupRule' => 'infrastructure/markup/rule/PhabricatorKeyboardRemarkupRule.php',
|
||||
|
@ -8800,6 +8801,7 @@ phutil_register_library_map(array(
|
|||
'PhabricatorJSONExportFormat' => 'PhabricatorExportFormat',
|
||||
'PhabricatorJavelinLinter' => 'ArcanistLinter',
|
||||
'PhabricatorJiraIssueHasObjectEdgeType' => 'PhabricatorEdgeType',
|
||||
'PhabricatorJupyterDocumentEngine' => 'PhabricatorDocumentEngine',
|
||||
'PhabricatorKeyValueDatabaseCache' => 'PhutilKeyValueCache',
|
||||
'PhabricatorKeyValueSerializingCacheProxy' => 'PhutilKeyValueCacheProxy',
|
||||
'PhabricatorKeyboardRemarkupRule' => 'PhutilRemarkupRule',
|
||||
|
|
|
@ -110,6 +110,15 @@ final class PhabricatorDocumentRef
|
|||
return (strpos($snippet, "\0") === false);
|
||||
}
|
||||
|
||||
public function isProbablyJSON() {
|
||||
if (!$this->isProbablyText()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$snippet = $this->getSnippet();
|
||||
return phutil_is_utf8($snippet);
|
||||
}
|
||||
|
||||
public function getSnippet() {
|
||||
if ($this->snippet === null) {
|
||||
$this->snippet = $this->loadData(null, (1024 * 1024 * 1));
|
||||
|
|
|
@ -0,0 +1,305 @@
|
|||
<?php
|
||||
|
||||
final class PhabricatorJupyterDocumentEngine
|
||||
extends PhabricatorDocumentEngine {
|
||||
|
||||
const ENGINEKEY = 'jupyter';
|
||||
|
||||
public function getViewAsLabel(PhabricatorDocumentRef $ref) {
|
||||
return pht('View as Jupyter Notebook');
|
||||
}
|
||||
|
||||
protected function getDocumentIconIcon(PhabricatorDocumentRef $ref) {
|
||||
return 'fa-sun-o';
|
||||
}
|
||||
|
||||
protected function getContentScore(PhabricatorDocumentRef $ref) {
|
||||
$name = $ref->getName();
|
||||
|
||||
if (preg_match('/\\.ipynb\z/i', $name)) {
|
||||
return 2000;
|
||||
}
|
||||
|
||||
return 500;
|
||||
}
|
||||
|
||||
protected function canRenderDocumentType(PhabricatorDocumentRef $ref) {
|
||||
return $ref->isProbablyJSON();
|
||||
}
|
||||
|
||||
protected function newDocumentContent(PhabricatorDocumentRef $ref) {
|
||||
$viewer = $this->getViewer();
|
||||
$content = $ref->loadData();
|
||||
|
||||
try {
|
||||
$data = phutil_json_decode($content);
|
||||
} catch (PhutilJSONParserException $ex) {
|
||||
return $this->newMessage(
|
||||
pht(
|
||||
'This is not a valid JSON document and can not be rendered as '.
|
||||
'a Jupyter notebook: %s.',
|
||||
$ex->getMessage()));
|
||||
}
|
||||
|
||||
if (!is_array($data)) {
|
||||
return $this->newMessage(
|
||||
pht(
|
||||
'This document does not encode a valid JSON object and can not '.
|
||||
'be rendered as a Jupyter notebook.'));
|
||||
}
|
||||
|
||||
|
||||
$nbformat = idx($data, 'nbformat');
|
||||
if (!strlen($nbformat)) {
|
||||
return $this->newMessage(
|
||||
pht(
|
||||
'This document is missing an "nbformat" field. Jupyter notebooks '.
|
||||
'must have this field.'));
|
||||
}
|
||||
|
||||
if ($nbformat !== 4) {
|
||||
return $this->newMessage(
|
||||
pht(
|
||||
'This Jupyter notebook uses an unsupported version of the file '.
|
||||
'format (found version %s, expected version 4).',
|
||||
$nbformat));
|
||||
}
|
||||
|
||||
$cells = idx($data, 'cells');
|
||||
if (!is_array($cells)) {
|
||||
return $this->newMessage(
|
||||
pht(
|
||||
'This Jupyter notebook does not specify a list of "cells".'));
|
||||
}
|
||||
|
||||
if (!$cells) {
|
||||
return $this->newMessage(
|
||||
pht(
|
||||
'This Jupyter notebook does not specify any notebook cells.'));
|
||||
}
|
||||
|
||||
$rows = array();
|
||||
foreach ($cells as $cell) {
|
||||
$rows[] = $this->renderJupyterCell($viewer, $cell);
|
||||
}
|
||||
|
||||
$notebook_table = phutil_tag(
|
||||
'table',
|
||||
array(
|
||||
'class' => 'jupyter-notebook',
|
||||
),
|
||||
$rows);
|
||||
|
||||
$container = phutil_tag(
|
||||
'div',
|
||||
array(
|
||||
'class' => 'document-engine-jupyter',
|
||||
),
|
||||
$notebook_table);
|
||||
|
||||
return $container;
|
||||
}
|
||||
|
||||
private function renderJupyterCell(
|
||||
PhabricatorUser $viewer,
|
||||
array $cell) {
|
||||
|
||||
list($label, $content) = $this->renderJupyterCellContent($viewer, $cell);
|
||||
|
||||
$label_cell = phutil_tag(
|
||||
'th',
|
||||
array(),
|
||||
$label);
|
||||
|
||||
$content_cell = phutil_tag(
|
||||
'td',
|
||||
array(),
|
||||
$content);
|
||||
|
||||
return phutil_tag(
|
||||
'tr',
|
||||
array(),
|
||||
array(
|
||||
$label_cell,
|
||||
$content_cell,
|
||||
));
|
||||
}
|
||||
|
||||
private function renderJupyterCellContent(
|
||||
PhabricatorUser $viewer,
|
||||
array $cell) {
|
||||
|
||||
$cell_type = idx($cell, 'cell_type');
|
||||
switch ($cell_type) {
|
||||
case 'markdown':
|
||||
return $this->newMarkdownCell($cell);
|
||||
case 'code':
|
||||
return $this->newCodeCell($cell);
|
||||
}
|
||||
|
||||
return $this->newRawCell(id(new PhutilJSON())->encodeFormatted($cell));
|
||||
}
|
||||
|
||||
private function newRawCell($content) {
|
||||
return array(
|
||||
null,
|
||||
phutil_tag(
|
||||
'div',
|
||||
array(
|
||||
'class' => 'jupyter-cell-raw PhabricatorMonospaced',
|
||||
),
|
||||
$content),
|
||||
);
|
||||
}
|
||||
|
||||
private function newMarkdownCell(array $cell) {
|
||||
$content = idx($cell, 'source');
|
||||
if (!is_array($content)) {
|
||||
$content = array();
|
||||
}
|
||||
|
||||
$content = implode('', $content);
|
||||
$content = phutil_escape_html_newlines($content);
|
||||
|
||||
return array(
|
||||
null,
|
||||
phutil_tag(
|
||||
'div',
|
||||
array(
|
||||
'class' => 'jupyter-cell-markdown',
|
||||
),
|
||||
$content),
|
||||
);
|
||||
}
|
||||
|
||||
private function newCodeCell(array $cell) {
|
||||
$execution_count = idx($cell, 'execution_count');
|
||||
if ($execution_count) {
|
||||
$label = 'In ['.$execution_count.']:';
|
||||
} else {
|
||||
$label = null;
|
||||
}
|
||||
|
||||
$content = idx($cell, 'source');
|
||||
if (!is_array($content)) {
|
||||
$content = array();
|
||||
}
|
||||
|
||||
$content = implode('', $content);
|
||||
|
||||
$content = PhabricatorSyntaxHighlighter::highlightWithLanguage(
|
||||
'python',
|
||||
$content);
|
||||
|
||||
$outputs = array();
|
||||
$output_list = idx($cell, 'outputs');
|
||||
if (is_array($output_list)) {
|
||||
foreach ($output_list as $output) {
|
||||
$outputs[] = $this->newOutput($output);
|
||||
}
|
||||
}
|
||||
|
||||
return array(
|
||||
$label,
|
||||
array(
|
||||
phutil_tag(
|
||||
'div',
|
||||
array(
|
||||
'class' => 'jupyter-cell-code PhabricatorMonospaced remarkup-code',
|
||||
),
|
||||
array(
|
||||
$content,
|
||||
)),
|
||||
$outputs,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
private function newOutput(array $output) {
|
||||
if (!is_array($output)) {
|
||||
return pht('<Invalid Output>');
|
||||
}
|
||||
|
||||
$classes = array(
|
||||
'jupyter-output',
|
||||
'PhabricatorMonospaced',
|
||||
);
|
||||
|
||||
$output_name = idx($output, 'name');
|
||||
switch ($output_name) {
|
||||
case 'stderr':
|
||||
$classes[] = 'jupyter-output-stderr';
|
||||
break;
|
||||
}
|
||||
|
||||
$output_type = idx($output, 'output_type');
|
||||
switch ($output_type) {
|
||||
case 'execute_result':
|
||||
case 'display_data':
|
||||
$data = idx($output, 'data');
|
||||
|
||||
$image_formats = array(
|
||||
'image/png',
|
||||
'image/jpeg',
|
||||
'image/jpg',
|
||||
'image/gif',
|
||||
);
|
||||
|
||||
foreach ($image_formats as $image_format) {
|
||||
if (!isset($data[$image_format])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$raw_data = $data[$image_format];
|
||||
if (!is_array($raw_data)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$raw_data = implode('', $raw_data);
|
||||
|
||||
$content = phutil_tag(
|
||||
'img',
|
||||
array(
|
||||
'src' => 'data:'.$image_format.';base64,'.$raw_data,
|
||||
));
|
||||
|
||||
break 2;
|
||||
}
|
||||
|
||||
if (isset($data['text/html'])) {
|
||||
$content = $data['text/html'];
|
||||
$classes[] = 'jupyter-output-html';
|
||||
break;
|
||||
}
|
||||
|
||||
if (isset($data['application/javascript'])) {
|
||||
$content = $data['application/javascript'];
|
||||
$classes[] = 'jupyter-output-html';
|
||||
break;
|
||||
}
|
||||
|
||||
if (isset($data['text/plain'])) {
|
||||
$content = $data['text/plain'];
|
||||
break;
|
||||
}
|
||||
|
||||
break;
|
||||
case 'stream':
|
||||
default:
|
||||
$content = idx($output, 'text');
|
||||
if (!is_array($content)) {
|
||||
$content = array();
|
||||
}
|
||||
$content = implode('', $content);
|
||||
break;
|
||||
}
|
||||
|
||||
return phutil_tag(
|
||||
'div',
|
||||
array(
|
||||
'class' => implode(' ', $classes),
|
||||
),
|
||||
$content);
|
||||
}
|
||||
|
||||
}
|
|
@ -257,3 +257,50 @@ div.phui-property-list-stacked .phui-property-list-properties
|
|||
.document-engine-pdf .phabricator-remarkup-embed-layout-link {
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.document-engine-jupyter {
|
||||
overflow: hidden;
|
||||
margin: 20px;
|
||||
}
|
||||
|
||||
.jupyter-cell-raw {
|
||||
white-space: pre-wrap;
|
||||
background: {$lightgreybackground};
|
||||
color: {$greytext};
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
.jupyter-cell-code {
|
||||
white-space: pre-wrap;
|
||||
background: {$lightgreybackground};
|
||||
padding: 8px;
|
||||
border: 1px solid {$lightgreyborder};
|
||||
border-radius: 2px;
|
||||
}
|
||||
|
||||
.jupyter-notebook > tbody > tr > th,
|
||||
.jupyter-notebook > tbody > tr > td {
|
||||
padding: 8px;
|
||||
}
|
||||
|
||||
.jupyter-notebook > tbody > tr > th {
|
||||
white-space: nowrap;
|
||||
text-align: right;
|
||||
min-width: 48px;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.jupyter-output {
|
||||
margin: 4px 0;
|
||||
padding: 8px;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-all;
|
||||
}
|
||||
|
||||
.jupyter-output-stderr {
|
||||
background: {$sh-redbackground};
|
||||
}
|
||||
|
||||
.jupyter-output-html {
|
||||
background: {$sh-indigobackground};
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue