Add a very rough, proof-of-concept Jupyter notebook document engine

Summary:
Depends on D19252. Ref T13105. This very roughly renders Jupyter notebooks.

It's probably better than showing the raw JSON, but not by much.

Test Plan:
  - Viewed various notebooks with various cell types, including markdown, code, stdout, stderr, images, HTML, and Javascript.
  - HTML and Javascript are not live-fired since they're wildly dangerous.

Maniphest Tasks: T13105

Differential Revision: https://secure.phabricator.com/D19253

resources/celerity/map.php

@@ -9,7 +9,7 @@ return array(
   'names' => array(
     'conpherence.pkg.css' => 'e68cf1fa',
     'conpherence.pkg.js' => '15191c65',
-    'core.pkg.css' => '2d73b2f3',
+    'core.pkg.css' => '7daac340',
     'core.pkg.js' => 'b9b4a943',
     'differential.pkg.css' => '113e692c',
     'differential.pkg.js' => 'f6d809c0',
@@ -168,7 +168,7 @@ return array(
     'rsrc/css/phui/phui-object-box.css' => '9cff003c',
     'rsrc/css/phui/phui-pager.css' => 'edcbc226',
     'rsrc/css/phui/phui-pinboard-view.css' => '2495140e',
-    'rsrc/css/phui/phui-property-list-view.css' => '47018d3c',
+    'rsrc/css/phui/phui-property-list-view.css' => '871f6815',
     'rsrc/css/phui/phui-remarkup-preview.css' => '54a34863',
     'rsrc/css/phui/phui-segment-bar-view.css' => 'b1d1b892',
     'rsrc/css/phui/phui-spacing.css' => '042804d6',
@@ -850,7 +850,7 @@ return array(
     'phui-oi-simple-ui-css' => 'a8beebea',
     'phui-pager-css' => 'edcbc226',
     'phui-pinboard-view-css' => '2495140e',
-    'phui-property-list-view-css' => '47018d3c',
+    'phui-property-list-view-css' => '871f6815',
     'phui-remarkup-preview-css' => '54a34863',
     'phui-segment-bar-view-css' => 'b1d1b892',
     'phui-spacing-css' => '042804d6',

src/__phutil_library_map__.php

@@ -3190,6 +3190,7 @@ phutil_register_library_map(array(
     'PhabricatorJSONExportFormat' => 'infrastructure/export/format/PhabricatorJSONExportFormat.php',
     'PhabricatorJavelinLinter' => 'infrastructure/lint/linter/PhabricatorJavelinLinter.php',
     'PhabricatorJiraIssueHasObjectEdgeType' => 'applications/doorkeeper/edge/PhabricatorJiraIssueHasObjectEdgeType.php',
+    'PhabricatorJupyterDocumentEngine' => 'applications/files/document/PhabricatorJupyterDocumentEngine.php',
     'PhabricatorKeyValueDatabaseCache' => 'applications/cache/PhabricatorKeyValueDatabaseCache.php',
     'PhabricatorKeyValueSerializingCacheProxy' => 'applications/cache/PhabricatorKeyValueSerializingCacheProxy.php',
     'PhabricatorKeyboardRemarkupRule' => 'infrastructure/markup/rule/PhabricatorKeyboardRemarkupRule.php',
@@ -8800,6 +8801,7 @@ phutil_register_library_map(array(
     'PhabricatorJSONExportFormat' => 'PhabricatorExportFormat',
     'PhabricatorJavelinLinter' => 'ArcanistLinter',
     'PhabricatorJiraIssueHasObjectEdgeType' => 'PhabricatorEdgeType',
+    'PhabricatorJupyterDocumentEngine' => 'PhabricatorDocumentEngine',
     'PhabricatorKeyValueDatabaseCache' => 'PhutilKeyValueCache',
     'PhabricatorKeyValueSerializingCacheProxy' => 'PhutilKeyValueCacheProxy',
     'PhabricatorKeyboardRemarkupRule' => 'PhutilRemarkupRule',

src/applications/files/document/PhabricatorDocumentRef.php

@@ -110,6 +110,15 @@ final class PhabricatorDocumentRef
     return (strpos($snippet, "\0") === false);
   }
 
+  public function isProbablyJSON() {
+    if (!$this->isProbablyText()) {
+      return false;
+    }
+
+    $snippet = $this->getSnippet();
+    return phutil_is_utf8($snippet);
+  }
+
   public function getSnippet() {
     if ($this->snippet === null) {
       $this->snippet = $this->loadData(null, (1024 * 1024 * 1));

src/applications/files/document/PhabricatorJupyterDocumentEngine.php

@@ -0,0 +1,305 @@
+<?php
+
+final class PhabricatorJupyterDocumentEngine
+  extends PhabricatorDocumentEngine {
+
+  const ENGINEKEY = 'jupyter';
+
+  public function getViewAsLabel(PhabricatorDocumentRef $ref) {
+    return pht('View as Jupyter Notebook');
+  }
+
+  protected function getDocumentIconIcon(PhabricatorDocumentRef $ref) {
+    return 'fa-sun-o';
+  }
+
+  protected function getContentScore(PhabricatorDocumentRef $ref) {
+    $name = $ref->getName();
+
+    if (preg_match('/\\.ipynb\z/i', $name)) {
+      return 2000;
+    }
+
+    return 500;
+  }
+
+  protected function canRenderDocumentType(PhabricatorDocumentRef $ref) {
+    return $ref->isProbablyJSON();
+  }
+
+  protected function newDocumentContent(PhabricatorDocumentRef $ref) {
+    $viewer = $this->getViewer();
+    $content = $ref->loadData();
+
+    try {
+      $data = phutil_json_decode($content);
+    } catch (PhutilJSONParserException $ex) {
+      return $this->newMessage(
+        pht(
+          'This is not a valid JSON document and can not be rendered as '.
+          'a Jupyter notebook: %s.',
+          $ex->getMessage()));
+    }
+
+    if (!is_array($data)) {
+      return $this->newMessage(
+        pht(
+          'This document does not encode a valid JSON object and can not '.
+          'be rendered as a Jupyter notebook.'));
+    }
+
+
+    $nbformat = idx($data, 'nbformat');
+    if (!strlen($nbformat)) {
+      return $this->newMessage(
+        pht(
+          'This document is missing an "nbformat" field. Jupyter notebooks '.
+          'must have this field.'));
+    }
+
+    if ($nbformat !== 4) {
+      return $this->newMessage(
+        pht(
+          'This Jupyter notebook uses an unsupported version of the file '.
+          'format (found version %s, expected version 4).',
+          $nbformat));
+    }
+
+    $cells = idx($data, 'cells');
+    if (!is_array($cells)) {
+      return $this->newMessage(
+        pht(
+          'This Jupyter notebook does not specify a list of "cells".'));
+    }
+
+    if (!$cells) {
+      return $this->newMessage(
+        pht(
+          'This Jupyter notebook does not specify any notebook cells.'));
+    }
+
+    $rows = array();
+    foreach ($cells as $cell) {
+      $rows[] = $this->renderJupyterCell($viewer, $cell);
+    }
+
+    $notebook_table = phutil_tag(
+      'table',
+      array(
+        'class' => 'jupyter-notebook',
+      ),
+      $rows);
+
+    $container = phutil_tag(
+      'div',
+      array(
+        'class' => 'document-engine-jupyter',
+      ),
+      $notebook_table);
+
+    return $container;
+  }
+
+  private function renderJupyterCell(
+    PhabricatorUser $viewer,
+    array $cell) {
+
+    list($label, $content) = $this->renderJupyterCellContent($viewer, $cell);
+
+    $label_cell = phutil_tag(
+      'th',
+      array(),
+      $label);
+
+    $content_cell = phutil_tag(
+      'td',
+      array(),
+      $content);
+
+    return phutil_tag(
+      'tr',
+      array(),
+      array(
+        $label_cell,
+        $content_cell,
+      ));
+  }
+
+  private function renderJupyterCellContent(
+    PhabricatorUser $viewer,
+    array $cell) {
+
+    $cell_type = idx($cell, 'cell_type');
+    switch ($cell_type) {
+      case 'markdown':
+        return $this->newMarkdownCell($cell);
+      case 'code':
+       return $this->newCodeCell($cell);
+    }
+
+    return $this->newRawCell(id(new PhutilJSON())->encodeFormatted($cell));
+  }
+
+  private function newRawCell($content) {
+    return array(
+      null,
+      phutil_tag(
+        'div',
+        array(
+          'class' => 'jupyter-cell-raw PhabricatorMonospaced',
+        ),
+        $content),
+    );
+  }
+
+  private function newMarkdownCell(array $cell) {
+    $content = idx($cell, 'source');
+    if (!is_array($content)) {
+      $content = array();
+    }
+
+    $content = implode('', $content);
+    $content = phutil_escape_html_newlines($content);
+
+    return array(
+      null,
+      phutil_tag(
+        'div',
+        array(
+          'class' => 'jupyter-cell-markdown',
+        ),
+        $content),
+    );
+  }
+
+  private function newCodeCell(array $cell) {
+    $execution_count = idx($cell, 'execution_count');
+    if ($execution_count) {
+      $label = 'In ['.$execution_count.']:';
+    } else {
+      $label = null;
+    }
+
+    $content = idx($cell, 'source');
+    if (!is_array($content)) {
+      $content = array();
+    }
+
+    $content = implode('', $content);
+
+    $content = PhabricatorSyntaxHighlighter::highlightWithLanguage(
+      'python',
+      $content);
+
+    $outputs = array();
+    $output_list = idx($cell, 'outputs');
+    if (is_array($output_list)) {
+      foreach ($output_list as $output) {
+        $outputs[] = $this->newOutput($output);
+      }
+    }
+
+    return array(
+      $label,
+      array(
+        phutil_tag(
+          'div',
+          array(
+            'class' => 'jupyter-cell-code PhabricatorMonospaced remarkup-code',
+          ),
+          array(
+            $content,
+          )),
+        $outputs,
+      ),
+    );
+  }
+
+  private function newOutput(array $output) {
+    if (!is_array($output)) {
+      return pht('<Invalid Output>');
+    }
+
+    $classes = array(
+      'jupyter-output',
+      'PhabricatorMonospaced',
+    );
+
+    $output_name = idx($output, 'name');
+    switch ($output_name) {
+      case 'stderr':
+        $classes[] = 'jupyter-output-stderr';
+        break;
+    }
+
+    $output_type = idx($output, 'output_type');
+    switch ($output_type) {
+      case 'execute_result':
+      case 'display_data':
+        $data = idx($output, 'data');
+
+        $image_formats = array(
+          'image/png',
+          'image/jpeg',
+          'image/jpg',
+          'image/gif',
+        );
+
+        foreach ($image_formats as $image_format) {
+          if (!isset($data[$image_format])) {
+            continue;
+          }
+
+          $raw_data = $data[$image_format];
+          if (!is_array($raw_data)) {
+            continue;
+          }
+
+          $raw_data = implode('', $raw_data);
+
+          $content = phutil_tag(
+            'img',
+            array(
+              'src' => 'data:'.$image_format.';base64,'.$raw_data,
+            ));
+
+          break 2;
+        }
+
+        if (isset($data['text/html'])) {
+          $content = $data['text/html'];
+          $classes[] = 'jupyter-output-html';
+          break;
+        }
+
+        if (isset($data['application/javascript'])) {
+          $content = $data['application/javascript'];
+          $classes[] = 'jupyter-output-html';
+          break;
+        }
+
+        if (isset($data['text/plain'])) {
+          $content = $data['text/plain'];
+          break;
+        }
+
+        break;
+      case 'stream':
+      default:
+        $content = idx($output, 'text');
+        if (!is_array($content)) {
+          $content = array();
+        }
+        $content = implode('', $content);
+        break;
+    }
+
+    return phutil_tag(
+      'div',
+      array(
+        'class' => implode(' ', $classes),
+      ),
+      $content);
+  }
+
+}

webroot/rsrc/css/phui/phui-property-list-view.css

@@ -257,3 +257,50 @@ div.phui-property-list-stacked .phui-property-list-properties
 .document-engine-pdf .phabricator-remarkup-embed-layout-link {
   text-align: left;
 }
+
+.document-engine-jupyter {
+  overflow: hidden;
+  margin: 20px;
+}
+
+.jupyter-cell-raw {
+  white-space: pre-wrap;
+  background: {$lightgreybackground};
+  color: {$greytext};
+  padding: 8px;
+}
+
+.jupyter-cell-code {
+  white-space: pre-wrap;
+  background: {$lightgreybackground};
+  padding: 8px;
+  border: 1px solid {$lightgreyborder};
+  border-radius: 2px;
+}
+
+.jupyter-notebook > tbody > tr > th,
+.jupyter-notebook > tbody > tr > td {
+  padding: 8px;
+}
+
+.jupyter-notebook > tbody > tr > th {
+  white-space: nowrap;
+  text-align: right;
+  min-width: 48px;
+  font-weight: bold;
+}
+
+.jupyter-output {
+  margin: 4px 0;
+  padding: 8px;
+  white-space: pre-wrap;
+  word-break: break-all;
+}
+
+.jupyter-output-stderr {
+  background: {$sh-redbackground};
+}
+
+.jupyter-output-html {
+  background: {$sh-indigobackground};
+}