Endpoint+controller for a remarkup image proxy

Summary:
Ref T4190. Currently only have the endpoint and controller working. I added caching so subsequent attempts to proxy the same image should result in the same redirect URL. Still need to:

- Write a remarkup rule that uses the endpoint

Test Plan: Hit /file/imageproxy/?uri=http://i.imgur.com/nTvVrYN.jpg and are served the picture

Reviewers: epriestley, #blessed_reviewers

Reviewed By: epriestley, #blessed_reviewers

Subscribers: Korvin, epriestley, yelirekim

Maniphest Tasks: T4190

Differential Revision: https://secure.phabricator.com/D16581

resources/sql/autopatches/20160921.fileexternalrequest.sql

@@ -0,0 +1,14 @@
+CREATE TABLE {$NAMESPACE}_file.file_externalrequest (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  filePHID VARBINARY(64),
+  ttl INT UNSIGNED NOT NULL,
+  uri LONGTEXT NOT NULL,
+  uriIndex BINARY(12) NOT NULL,
+  isSuccessful BOOL NOT NULL,
+  responseMessage LONGTEXT,
+  dateCreated INT UNSIGNED NOT NULL,
+  dateModified INT UNSIGNED NOT NULL,
+  UNIQUE KEY `key_uriindex` (uriIndex),
+  KEY `key_ttl` (ttl),
+  KEY `key_file` (filePHID)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

src/__phutil_library_map__.php

@@ -2553,10 +2553,13 @@ phutil_register_library_map(array(
     'PhabricatorFileDropUploadController' => 'applications/files/controller/PhabricatorFileDropUploadController.php',
     'PhabricatorFileEditController' => 'applications/files/controller/PhabricatorFileEditController.php',
     'PhabricatorFileEditor' => 'applications/files/editor/PhabricatorFileEditor.php',
+    'PhabricatorFileExternalRequest' => 'applications/files/storage/PhabricatorFileExternalRequest.php',
+    'PhabricatorFileExternalRequestGarbageCollector' => 'applications/files/garbagecollector/PhabricatorFileExternalRequestGarbageCollector.php',
     'PhabricatorFileFilePHIDType' => 'applications/files/phid/PhabricatorFileFilePHIDType.php',
     'PhabricatorFileHasObjectEdgeType' => 'applications/files/edge/PhabricatorFileHasObjectEdgeType.php',
     'PhabricatorFileIconSetSelectController' => 'applications/files/controller/PhabricatorFileIconSetSelectController.php',
     'PhabricatorFileImageMacro' => 'applications/macro/storage/PhabricatorFileImageMacro.php',
+    'PhabricatorFileImageProxyController' => 'applications/files/controller/PhabricatorFileImageProxyController.php',
     'PhabricatorFileImageTransform' => 'applications/files/transform/PhabricatorFileImageTransform.php',
     'PhabricatorFileInfoController' => 'applications/files/controller/PhabricatorFileInfoController.php',
     'PhabricatorFileLinkView' => 'view/layout/PhabricatorFileLinkView.php',
@@ -7368,6 +7371,11 @@ phutil_register_library_map(array(
     'PhabricatorFileDropUploadController' => 'PhabricatorFileController',
     'PhabricatorFileEditController' => 'PhabricatorFileController',
     'PhabricatorFileEditor' => 'PhabricatorApplicationTransactionEditor',
+    'PhabricatorFileExternalRequest' => array(
+      'PhabricatorFileDAO',
+      'PhabricatorDestructibleInterface',
+    ),
+    'PhabricatorFileExternalRequestGarbageCollector' => 'PhabricatorGarbageCollector',
     'PhabricatorFileFilePHIDType' => 'PhabricatorPHIDType',
     'PhabricatorFileHasObjectEdgeType' => 'PhabricatorEdgeType',
     'PhabricatorFileIconSetSelectController' => 'PhabricatorFileController',
@@ -7379,6 +7387,7 @@ phutil_register_library_map(array(
       'PhabricatorTokenReceiverInterface',
       'PhabricatorPolicyInterface',
     ),
+    'PhabricatorFileImageProxyController' => 'PhabricatorFileController',
     'PhabricatorFileImageTransform' => 'PhabricatorFileTransform',
     'PhabricatorFileInfoController' => 'PhabricatorFileController',
     'PhabricatorFileLinkView' => 'AphrontView',

src/applications/files/application/PhabricatorFilesApplication.php

@@ -78,7 +78,7 @@ final class PhabricatorFilesApplication extends PhabricatorApplication {
         'delete/(?P<id>[1-9]\d*)/' => 'PhabricatorFileDeleteController',
         'edit/(?P<id>[1-9]\d*)/' => 'PhabricatorFileEditController',
         'info/(?P<phid>[^/]+)/' => 'PhabricatorFileInfoController',
-        'proxy/' => 'PhabricatorFileProxyController',
+        'imageproxy/' => 'PhabricatorFileImageProxyController',
         'transforms/(?P<id>[1-9]\d*)/' =>
           'PhabricatorFileTransformListController',
         'uploaddialog/(?P<single>single/)?'

src/applications/files/controller/PhabricatorFileImageProxyController.php

@@ -0,0 +1,118 @@
+<?php
+
+final class PhabricatorFileImageProxyController
+  extends PhabricatorFileController {
+
+  public function shouldAllowPublic() {
+    return true;
+  }
+
+  public function handleRequest(AphrontRequest $request) {
+
+    $show_prototypes = PhabricatorEnv::getEnvConfig(
+      'phabricator.show-prototypes');
+    if (!$show_prototypes) {
+      throw new Exception(
+        pht('Show prototypes is disabled.
+          Set `phabricator.show-prototypes` to `true` to use the image proxy'));
+    }
+
+    $viewer = $request->getViewer();
+    $img_uri = $request->getStr('uri');
+
+    // Validate the URI before doing anything
+    PhabricatorEnv::requireValidRemoteURIForLink($img_uri);
+    $uri = new PhutilURI($img_uri);
+    $proto = $uri->getProtocol();
+    if (!in_array($proto, array('http', 'https'))) {
+      throw new Exception(
+        pht('The provided image URI must be either http or https'));
+    }
+
+    // Check if we already have the specified image URI downloaded
+    $cached_request = id(new PhabricatorFileExternalRequest())->loadOneWhere(
+      'uriIndex = %s',
+      PhabricatorHash::digestForIndex($img_uri));
+
+    if ($cached_request) {
+      return $this->getExternalResponse($cached_request);
+    }
+
+    $ttl = PhabricatorTime::getNow() + phutil_units('7 days in seconds');
+    $external_request = id(new PhabricatorFileExternalRequest())
+      ->setURI($img_uri)
+      ->setTTL($ttl);
+
+    $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
+    // Cache missed so we'll need to validate and download the image
+    try {
+      // Rate limit outbound fetches to make this mechanism less useful for
+      // scanning networks and ports.
+      PhabricatorSystemActionEngine::willTakeAction(
+        array($viewer->getPHID()),
+        new PhabricatorFilesOutboundRequestAction(),
+        1);
+
+      $file = PhabricatorFile::newFromFileDownload(
+        $uri,
+        array(
+          'viewPolicy' => PhabricatorPolicies::POLICY_NOONE,
+          'canCDN' => true,
+        ));
+      if (!$file->isViewableImage()) {
+        $mime_type = $file->getMimeType();
+        $engine = new PhabricatorDestructionEngine();
+        $engine->destroyObject($file);
+        $file = null;
+        throw new Exception(
+          pht(
+            'The URI "%s" does not correspond to a valid image file, got '.
+            'a file with MIME type "%s". You must specify the URI of a '.
+            'valid image file.',
+            $uri,
+            $mime_type));
+      } else {
+        $file->save();
+      }
+
+      $external_request->setIsSuccessful(true)
+        ->setFilePHID($file->getPHID())
+        ->save();
+      unset($unguarded);
+      return $this->getExternalResponse($external_request);
+    } catch (HTTPFutureHTTPResponseStatus $status) {
+      $external_request->setIsSuccessful(false)
+        ->setResponseMessage($status->getMessage())
+        ->save();
+      return $this->getExternalResponse($external_request);
+    } catch (Exception $ex) {
+      // Not actually saving the request in this case
+      $external_request->setResponseMessage($ex->getMessage());
+      return $this->getExternalResponse($external_request);
+    }
+  }
+
+  private function getExternalResponse(
+    PhabricatorFileExternalRequest $request) {
+    if ($request->getIsSuccessful()) {
+      $file = id(new PhabricatorFileQuery())
+        ->setViewer(PhabricatorUser::getOmnipotentUser())
+        ->withPHIDs(array($request->getFilePHID()))
+        ->executeOne();
+      if (!file) {
+        throw new Exception(pht(
+          'The underlying file does not exist, but the cached request was '.
+          'successful. This likely means the file record was manually deleted '.
+          'by an administrator.'));
+      }
+      return id(new AphrontRedirectResponse())
+        ->setIsExternal(true)
+        ->setURI($file->getViewURI());
+    } else {
+      throw new Exception(pht(
+        "The request to get the external file from '%s' was unsuccessful:\n %s",
+        $request->getURI(),
+        $request->getResponseMessage()));
+    }
+  }
+}

src/applications/files/garbagecollector/PhabricatorFileExternalRequestGarbageCollector.php

@@ -0,0 +1,28 @@
+<?php
+
+final class PhabricatorFileExternalRequestGarbageCollector
+  extends PhabricatorGarbageCollector {
+
+  const COLLECTORCONST = 'files.externalttl';
+
+  public function getCollectorName() {
+    return pht('External Requests (TTL)');
+  }
+
+  public function hasAutomaticPolicy() {
+    return true;
+  }
+
+  protected function collectGarbage() {
+    $file_requests = id(new PhabricatorFileExternalRequest())->loadAllWhere(
+      'ttl < %d LIMIT 100',
+      PhabricatorTime::getNow());
+    $engine = new PhabricatorDestructionEngine();
+    foreach ($file_requests as $request) {
+      $engine->destroyObject($request);
+    }
+
+    return (count($file_requests) == 100);
+  }
+
+}

src/applications/files/storage/PhabricatorFileExternalRequest.php

@@ -0,0 +1,63 @@
+<?php
+
+final class PhabricatorFileExternalRequest extends PhabricatorFileDAO
+  implements
+    PhabricatorDestructibleInterface {
+
+  protected $uri;
+  protected $uriIndex;
+  protected $ttl;
+  protected $filePHID;
+  protected $isSuccessful;
+  protected $responseMessage;
+
+  protected function getConfiguration() {
+    return array(
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'uri' => 'text',
+        'uriIndex' => 'bytes12',
+        'ttl' => 'epoch',
+        'filePHID' => 'phid?',
+        'isSuccessful' => 'bool',
+        'responseMessage' => 'text?',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_uriindex' => array(
+          'columns' => array('uriIndex'),
+          'unique' => true,
+        ),
+        'key_ttl' => array(
+          'columns' => array('ttl'),
+        ),
+        'key_file' => array(
+          'columns' => array('filePHID'),
+        ),
+      ),
+    ) + parent::getConfiguration();
+  }
+
+  public function save() {
+    $hash = PhabricatorHash::digestForIndex($this->getURI());
+    $this->setURIIndex($hash);
+    return parent::save();
+  }
+
+/* -(  PhabricatorDestructibleInterface  )----------------------------------- */
+
+  public function destroyObjectPermanently(
+    PhabricatorDestructionEngine $engine) {
+
+    $file_phid = $this->getFilePHID();
+    if ($file_phid) {
+      $file = id(new PhabricatorFileQuery())
+        ->setViewer($engine->getViewer())
+        ->withPHIDs(array($file_phid))
+        ->executeOne();
+      if ($file) {
+        $engine->destroyObject($file);
+      }
+    }
+    $this->delete();
+  }
+
+}