phorge-phorge/src/applications/files/storage/PhabricatorFile.php

<?php

/**
 * @group file
 */
final class PhabricatorFile extends PhabricatorFileDAO
  implements
    PhabricatorTokenReceiverInterface,
    PhabricatorSubscribableInterface,
    PhabricatorPolicyInterface {

  const STORAGE_FORMAT_RAW  = 'raw';

  const METADATA_IMAGE_WIDTH  = 'width';
  const METADATA_IMAGE_HEIGHT = 'height';

  protected $phid;
  protected $name;
  protected $mimeType;
  protected $byteSize;
  protected $authorPHID;
  protected $secretKey;
  protected $contentHash;
  protected $metadata = array();
  protected $mailKey;

  protected $storageEngine;
  protected $storageFormat;
  protected $storageHandle;

  protected $ttl;
  protected $isExplicitUpload = 1;

  public function getConfiguration() {
    return array(
      self::CONFIG_AUX_PHID => true,
      self::CONFIG_SERIALIZATION => array(
        'metadata' => self::SERIALIZATION_JSON,
      ),
    ) + parent::getConfiguration();
  }

  public function generatePHID() {
    return PhabricatorPHID::generateNewPHID(
      PhabricatorFilePHIDTypeFile::TYPECONST);
  }

  public function save() {
    if (!$this->getSecretKey()) {
      $this->setSecretKey($this->generateSecretKey());
    }
    if (!$this->getMailKey()) {
      $this->setMailKey(Filesystem::readRandomCharacters(20));
    }
    return parent::save();
  }

  public static function readUploadedFileData($spec) {
    if (!$spec) {
      throw new Exception("No file was uploaded!");
    }

    $err = idx($spec, 'error');
    if ($err) {
      throw new PhabricatorFileUploadException($err);
    }

    $tmp_name = idx($spec, 'tmp_name');
    $is_valid = @is_uploaded_file($tmp_name);
    if (!$is_valid) {
      throw new Exception("File is not an uploaded file.");
    }

    $file_data = Filesystem::readFile($tmp_name);
    $file_size = idx($spec, 'size');

    if (strlen($file_data) != $file_size) {
      throw new Exception("File size disagrees with uploaded size.");
    }

    self::validateFileSize(strlen($file_data));

    return $file_data;
  }

  public static function newFromPHPUpload($spec, array $params = array()) {
    $file_data = self::readUploadedFileData($spec);

    $file_name = nonempty(
      idx($params, 'name'),
      idx($spec,   'name'));
    $params = array(
      'name' => $file_name,
    ) + $params;

    return self::newFromFileData($file_data, $params);
  }

  public static function newFromXHRUpload($data, array $params = array()) {
    self::validateFileSize(strlen($data));
    return self::newFromFileData($data, $params);
  }

  private static function validateFileSize($size) {
    $limit = PhabricatorEnv::getEnvConfig('storage.upload-size-limit');
    if (!$limit) {
      return;
    }

    $limit = phabricator_parse_bytes($limit);
    if ($size > $limit) {
      throw new PhabricatorFileUploadException(-1000);
    }
  }


  /**
   * Given a block of data, try to load an existing file with the same content
   * if one exists. If it does not, build a new file.
   *
   * This method is generally used when we have some piece of semi-trusted data
   * like a diff or a file from a repository that we want to show to the user.
   * We can't just dump it out because it may be dangerous for any number of
   * reasons; instead, we need to serve it through the File abstraction so it
   * ends up on the CDN domain if one is configured and so on. However, if we
   * simply wrote a new file every time we'd potentially end up with a lot
   * of redundant data in file storage.
   *
   * To solve these problems, we use file storage as a cache and reuse the
   * same file again if we've previously written it.
   *
   * NOTE: This method unguards writes.
   *
   * @param string  Raw file data.
   * @param dict    Dictionary of file information.
   */
  public static function buildFromFileDataOrHash(
    $data,
    array $params = array()) {

    $file = id(new PhabricatorFile())->loadOneWhere(
      'name = %s AND contentHash = %s LIMIT 1',
      self::normalizeFileName(idx($params, 'name')),
      self::hashFileContent($data));

    if (!$file) {
      $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
      $file = PhabricatorFile::newFromFileData($data, $params);
      unset($unguarded);
    }

    return $file;
  }

  public static function newFileFromContentHash($hash, $params) {

    // Check to see if a file with same contentHash exist
    $file = id(new PhabricatorFile())->loadOneWhere(
      'contentHash = %s LIMIT 1', $hash);

    if ($file) {
      // copy storageEngine, storageHandle, storageFormat
      $copy_of_storage_engine = $file->getStorageEngine();
      $copy_of_storage_handle = $file->getStorageHandle();
      $copy_of_storage_format = $file->getStorageFormat();
      $copy_of_byteSize = $file->getByteSize();
      $copy_of_mimeType = $file->getMimeType();

      $file_name = idx($params, 'name');
      $file_name = self::normalizeFileName($file_name);
      $file_ttl = idx($params, 'ttl');
      $authorPHID = idx($params, 'authorPHID');

      $new_file = new  PhabricatorFile();

      $new_file->setName($file_name);
      $new_file->setByteSize($copy_of_byteSize);
      $new_file->setAuthorPHID($authorPHID);
      $new_file->setTtl($file_ttl);

      $new_file->setContentHash($hash);
      $new_file->setStorageEngine($copy_of_storage_engine);
      $new_file->setStorageHandle($copy_of_storage_handle);
      $new_file->setStorageFormat($copy_of_storage_format);
      $new_file->setMimeType($copy_of_mimeType);
      $new_file->copyDimensions($file);

      $new_file->save();

      return $new_file;
    }

    return $file;
  }

  private static function buildFromFileData($data, array $params = array()) {
    $selector = PhabricatorEnv::newObjectFromConfig('storage.engine-selector');

    if (isset($params['storageEngines'])) {
      $engines = $params['storageEngines'];
    } else {
      $selector = PhabricatorEnv::newObjectFromConfig(
        'storage.engine-selector');
      $engines = $selector->selectStorageEngines($data, $params);
    }

    assert_instances_of($engines, 'PhabricatorFileStorageEngine');
    if (!$engines) {
      throw new Exception("No valid storage engines are available!");
    }

    $file = new PhabricatorFile();

    $data_handle = null;
    $engine_identifier = null;
    $exceptions = array();
    foreach ($engines as $engine) {
      $engine_class = get_class($engine);
      try {
        list($engine_identifier, $data_handle) = $file->writeToEngine(
          $engine,
          $data,
          $params);

        // We stored the file somewhere so stop trying to write it to other
        // places.
        break;
      } catch (PhabricatorFileStorageConfigurationException $ex) {
        // If an engine is outright misconfigured (or misimplemented), raise
        // that immediately since it probably needs attention.
        throw $ex;
      } catch (Exception $ex) {
        phlog($ex);

        // If an engine doesn't work, keep trying all the other valid engines
        // in case something else works.
        $exceptions[$engine_class] = $ex;
      }
    }

    if (!$data_handle) {
      throw new PhutilAggregateException(
        "All storage engines failed to write file:",
        $exceptions);
    }

    $file_name = idx($params, 'name');
    $file_name = self::normalizeFileName($file_name);
    $file_ttl = idx($params, 'ttl');

    // If for whatever reason, authorPHID isn't passed as a param
    // (always the case with newFromFileDownload()), store a ''
    $authorPHID = idx($params, 'authorPHID');

    $file->setName($file_name);
    $file->setByteSize(strlen($data));
    $file->setAuthorPHID($authorPHID);
    $file->setTtl($file_ttl);
    $file->setContentHash(self::hashFileContent($data));

    $file->setStorageEngine($engine_identifier);
    $file->setStorageHandle($data_handle);

    // TODO: This is probably YAGNI, but allows for us to do encryption or
    // compression later if we want.
    $file->setStorageFormat(self::STORAGE_FORMAT_RAW);
    $file->setIsExplicitUpload(idx($params, 'isExplicitUpload') ? 1 : 0);

    if (isset($params['mime-type'])) {
      $file->setMimeType($params['mime-type']);
    } else {
      $tmp = new TempFile();
      Filesystem::writeFile($tmp, $data);
      $file->setMimeType(Filesystem::getMimeType($tmp));
    }

    try {
      $file->updateDimensions(false);
    } catch (Exception $ex) {
      // Do nothing
    }

    $file->save();

    return $file;
  }

  public static function newFromFileData($data, array $params = array()) {
    $hash = self::hashFileContent($data);
    $file = self::newFileFromContentHash($hash, $params);

    if ($file) {
      return $file;
    }

    return self::buildFromFileData($data, $params);
  }

  public function migrateToEngine(PhabricatorFileStorageEngine $engine) {
    if (!$this->getID() || !$this->getStorageHandle()) {
      throw new Exception(
        "You can not migrate a file which hasn't yet been saved.");
    }

    $data = $this->loadFileData();
    $params = array(
      'name' => $this->getName(),
    );

    list($new_identifier, $new_handle) = $this->writeToEngine(
      $engine,
      $data,
      $params);

    $old_engine = $this->instantiateStorageEngine();
    $old_handle = $this->getStorageHandle();

    $this->setStorageEngine($new_identifier);
    $this->setStorageHandle($new_handle);
    $this->save();

    $old_engine->deleteFile($old_handle);

    return $this;
  }

  private function writeToEngine(
    PhabricatorFileStorageEngine $engine,
    $data,
    array $params) {

    $engine_class = get_class($engine);

    $data_handle = $engine->writeFile($data, $params);

    if (!$data_handle || strlen($data_handle) > 255) {
      // This indicates an improperly implemented storage engine.
      throw new PhabricatorFileStorageConfigurationException(
        "Storage engine '{$engine_class}' executed writeFile() but did ".
        "not return a valid handle ('{$data_handle}') to the data: it ".
        "must be nonempty and no longer than 255 characters.");
    }

    $engine_identifier = $engine->getEngineIdentifier();
    if (!$engine_identifier || strlen($engine_identifier) > 32) {
      throw new PhabricatorFileStorageConfigurationException(
        "Storage engine '{$engine_class}' returned an improper engine ".
        "identifier '{$engine_identifier}': it must be nonempty ".
        "and no longer than 32 characters.");
    }

    return array($engine_identifier, $data_handle);
  }


  public static function newFromFileDownload($uri, array $params = array()) {
    // Make sure we're allowed to make a request first
    if (!PhabricatorEnv::getEnvConfig('security.allow-outbound-http')) {
      throw new Exception("Outbound HTTP requests are disabled!");
    }

    $uri = new PhutilURI($uri);

    $protocol = $uri->getProtocol();
    switch ($protocol) {
      case 'http':
      case 'https':
        break;
      default:
        // Make sure we are not accessing any file:// URIs or similar.
        return null;
    }

    $timeout = 5;

    list($file_data) = id(new HTTPSFuture($uri))
        ->setTimeout($timeout)
        ->resolvex();

    $params = $params + array(
      'name' => basename($uri),
    );

    return self::newFromFileData($file_data, $params);
  }

  public static function normalizeFileName($file_name) {
    $pattern = "@[\\x00-\\x19#%&+!~'\$\"\/=\\\\?<> ]+@";
    $file_name = preg_replace($pattern, '_', $file_name);
    $file_name = preg_replace('@_+@', '_', $file_name);
    $file_name = trim($file_name, '_');

    $disallowed_filenames = array(
      '.'  => 'dot',
      '..' => 'dotdot',
      ''   => 'file',
    );
    $file_name = idx($disallowed_filenames, $file_name, $file_name);

    return $file_name;
  }

  public function delete() {

    // We want to delete all the rows which mark this file as the transformation
    // of some other file (since we're getting rid of it). We also delete all
    // the transformations of this file, so that a user who deletes an image
    // doesn't need to separately hunt down and delete a bunch of thumbnails and
    // resizes of it.

    $outbound_xforms = id(new PhabricatorFileQuery())
      ->setViewer(PhabricatorUser::getOmnipotentUser())
      ->withTransforms(
        array(
          array(
            'originalPHID' => $this->getPHID(),
            'transform'    => true,
          ),
        ))
      ->execute();

    foreach ($outbound_xforms as $outbound_xform) {
      $outbound_xform->delete();
    }

    $inbound_xforms = id(new PhabricatorTransformedFile())->loadAllWhere(
      'transformedPHID = %s',
      $this->getPHID());

    $this->openTransaction();
      foreach ($inbound_xforms as $inbound_xform) {
        $inbound_xform->delete();
      }
      $ret = parent::delete();
    $this->saveTransaction();

    // Check to see if other files are using storage
    $other_file = id(new PhabricatorFile())->loadAllWhere(
      'storageEngine = %s AND storageHandle = %s AND
      storageFormat = %s AND id != %d LIMIT 1',
      $this->getStorageEngine(),
      $this->getStorageHandle(),
      $this->getStorageFormat(),
      $this->getID());

    // If this is the only file using the storage, delete storage
    if (!$other_file) {
      $engine = $this->instantiateStorageEngine();
      try {
        $engine->deleteFile($this->getStorageHandle());
      } catch (Exception $ex) {
        // In the worst case, we're leaving some data stranded in a storage
        // engine, which is fine.
        phlog($ex);
      }
    }

    return $ret;
  }

  public static function hashFileContent($data) {
    return sha1($data);
  }

  public function loadFileData() {

    $engine = $this->instantiateStorageEngine();
    $data = $engine->readFile($this->getStorageHandle());

    switch ($this->getStorageFormat()) {
      case self::STORAGE_FORMAT_RAW:
        $data = $data;
        break;
      default:
        throw new Exception("Unknown storage format.");
    }

    return $data;
  }

  public function getViewURI() {
    if (!$this->getPHID()) {
      throw new Exception(
        "You must save a file before you can generate a view URI.");
    }

    $name = phutil_escape_uri($this->getName());

    $path = '/file/data/'.$this->getSecretKey().'/'.$this->getPHID().'/'.$name;
    return PhabricatorEnv::getCDNURI($path);
  }

  public function getInfoURI() {
    return '/file/info/'.$this->getPHID().'/';
  }

  public function getBestURI() {
    if ($this->isViewableInBrowser()) {
      return $this->getViewURI();
    } else {
      return $this->getInfoURI();
    }
  }

  public function getDownloadURI() {
    $uri = id(new PhutilURI($this->getViewURI()))
      ->setQueryParam('download', true);
    return (string) $uri;
  }

  public function getProfileThumbURI() {
    $path = '/file/xform/thumb-profile/'.$this->getPHID().'/'
      .$this->getSecretKey().'/';
    return PhabricatorEnv::getCDNURI($path);
  }

  public function getThumb60x45URI() {
    $path = '/file/xform/thumb-60x45/'.$this->getPHID().'/'
      .$this->getSecretKey().'/';
    return PhabricatorEnv::getCDNURI($path);
  }

  public function getThumb160x120URI() {
    $path = '/file/xform/thumb-160x120/'.$this->getPHID().'/'
      .$this->getSecretKey().'/';
    return PhabricatorEnv::getCDNURI($path);
  }

  public function getPreview140URI() {
    $path = '/file/xform/preview-140/'.$this->getPHID().'/'
      .$this->getSecretKey().'/';
    return PhabricatorEnv::getCDNURI($path);
  }

  public function getPreview220URI() {
    $path = '/file/xform/preview-220/'.$this->getPHID().'/'
      .$this->getSecretKey().'/';
    return PhabricatorEnv::getCDNURI($path);
  }

  public function getThumb220x165URI() {
    $path = '/file/xform/thumb-220x165/'.$this->getPHID().'/'
      .$this->getSecretKey().'/';
    return PhabricatorEnv::getCDNURI($path);
  }

  public function getThumb280x210URI() {
    $path = '/file/xform/thumb-280x210/'.$this->getPHID().'/'
      .$this->getSecretKey().'/';
    return PhabricatorEnv::getCDNURI($path);
  }

  public function isViewableInBrowser() {
    return ($this->getViewableMimeType() !== null);
  }

  public function isViewableImage() {
    if (!$this->isViewableInBrowser()) {
      return false;
    }

    $mime_map = PhabricatorEnv::getEnvConfig('files.image-mime-types');
    $mime_type = $this->getMimeType();
    return idx($mime_map, $mime_type);
  }

  public function isTransformableImage() {

    // NOTE: The way the 'gd' extension works in PHP is that you can install it
    // with support for only some file types, so it might be able to handle
    // PNG but not JPEG. Try to generate thumbnails for whatever we can. Setup
    // warns you if you don't have complete support.

    $matches = null;
    $ok = preg_match(
      '@^image/(gif|png|jpe?g)@',
      $this->getViewableMimeType(),
      $matches);
    if (!$ok) {
      return false;
    }

    switch ($matches[1]) {
      case 'jpg';
      case 'jpeg':
        return function_exists('imagejpeg');
        break;
      case 'png':
        return function_exists('imagepng');
        break;
      case 'gif':
        return function_exists('imagegif');
        break;
      default:
        throw new Exception('Unknown type matched as image MIME type.');
    }
  }

  public static function getTransformableImageFormats() {
    $supported = array();

    if (function_exists('imagejpeg')) {
      $supported[] = 'jpg';
    }

    if (function_exists('imagepng')) {
      $supported[] = 'png';
    }

    if (function_exists('imagegif')) {
      $supported[] = 'gif';
    }

    return $supported;
  }

  protected function instantiateStorageEngine() {
    return self::buildEngine($this->getStorageEngine());
  }

  public static function buildEngine($engine_identifier) {
    $engines = self::buildAllEngines();
    foreach ($engines as $engine) {
      if ($engine->getEngineIdentifier() == $engine_identifier) {
        return $engine;
      }
    }

    throw new Exception(
      "Storage engine '{$engine_identifier}' could not be located!");
  }

  public static function buildAllEngines() {
    $engines = id(new PhutilSymbolLoader())
      ->setType('class')
      ->setConcreteOnly(true)
      ->setAncestorClass('PhabricatorFileStorageEngine')
      ->selectAndLoadSymbols();

    $results = array();
    foreach ($engines as $engine_class) {
      $results[] = newv($engine_class['name'], array());
    }

    return $results;
  }

  public function getViewableMimeType() {
    $mime_map = PhabricatorEnv::getEnvConfig('files.viewable-mime-types');

    $mime_type = $this->getMimeType();
    $mime_parts = explode(';', $mime_type);
    $mime_type = trim(reset($mime_parts));

    return idx($mime_map, $mime_type);
  }

  public function getDisplayIconForMimeType() {
    $mime_map = PhabricatorEnv::getEnvConfig('files.icon-mime-types');
    $mime_type = $this->getMimeType();
    return idx($mime_map, $mime_type, 'docs_file');
  }

  public function validateSecretKey($key) {
    return ($key == $this->getSecretKey());
  }

  public function generateSecretKey() {
    return Filesystem::readRandomCharacters(20);
  }

  public function updateDimensions($save = true) {
    if (!$this->isViewableImage()) {
      throw new Exception(
        "This file is not a viewable image.");
    }

    if (!function_exists("imagecreatefromstring")) {
      throw new Exception(
        "Cannot retrieve image information.");
    }

    $data = $this->loadFileData();

    $img = imagecreatefromstring($data);
    if ($img === false) {
      throw new Exception(
        "Error when decoding image.");
    }

    $this->metadata[self::METADATA_IMAGE_WIDTH] = imagesx($img);
    $this->metadata[self::METADATA_IMAGE_HEIGHT] = imagesy($img);

    if ($save) {
      $this->save();
    }

    return $this;
  }

  public function copyDimensions(PhabricatorFile $file) {
    $metadata = $file->getMetadata();
    $width = idx($metadata, self::METADATA_IMAGE_WIDTH);
    if ($width) {
      $this->metadata[self::METADATA_IMAGE_WIDTH] = $width;
    }
    $height = idx($metadata, self::METADATA_IMAGE_HEIGHT);
    if ($height) {
      $this->metadata[self::METADATA_IMAGE_HEIGHT] = $height;
    }

    return $this;
  }

  public static function getMetadataName($metadata) {
    switch ($metadata) {
      case self::METADATA_IMAGE_WIDTH:
        $name = pht('Width');
        break;
      case self::METADATA_IMAGE_HEIGHT:
        $name = pht('Height');
        break;
      default:
        $name = ucfirst($metadata);
        break;
    }

    return $name;
  }


  /**
   * Load (or build) the {@class:PhabricatorFile} objects for builtin file
   * resources. The builtin mechanism allows files shipped with Phabricator
   * to be treated like normal files so that APIs do not need to special case
   * things like default images or deleted files.
   *
   * Builtins are located in `resources/builtin/` and identified by their
   * name.
   *
   * @param  PhabricatorUser                Viewing user.
   * @param  list<string>                   List of builtin file names.
   * @return dict<string, PhabricatorFile>  Dictionary of named builtins.
   */
  public static function loadBuiltins(PhabricatorUser $user, array $names) {
    $specs = array();
    foreach ($names as $name) {
      $specs[] = array(
        'originalPHID' => PhabricatorPHIDConstants::PHID_VOID,
        'transform'    => 'builtin:'.$name,
      );
    }

    $files = id(new PhabricatorFileQuery())
      ->setViewer($user)
      ->withTransforms($specs)
      ->execute();

    $files = mpull($files, null, 'getName');

    $root = dirname(phutil_get_library_root('phabricator'));
    $root = $root.'/resources/builtin/';

    $build = array();
    foreach ($names as $name) {
      if (isset($files[$name])) {
        continue;
      }

      // This is just a sanity check to prevent loading arbitrary files.
      if (basename($name) != $name) {
        throw new Exception("Invalid builtin name '{$name}'!");
      }

      $path = $root.$name;

      if (!Filesystem::pathExists($path)) {
        throw new Exception("Builtin '{$path}' does not exist!");
      }

      $data = Filesystem::readFile($path);
      $params = array(
        'name' => $name,
        'ttl'  => time() + (60 * 60 * 24 * 7),
      );

      $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
        $file = PhabricatorFile::newFromFileData($data, $params);
        $xform = id(new PhabricatorTransformedFile())
          ->setOriginalPHID(PhabricatorPHIDConstants::PHID_VOID)
          ->setTransform('builtin:'.$name)
          ->setTransformedPHID($file->getPHID())
          ->save();
      unset($unguarded);

      $files[$name] = $file;
    }

    return $files;
  }


  /**
   * Convenience wrapper for @{method:loadBuiltins}.
   *
   * @param PhabricatorUser   Viewing user.
   * @param string            Single builtin name to load.
   * @return PhabricatorFile  Corresponding builtin file.
   */
  public static function loadBuiltin(PhabricatorUser $user, $name) {
    return idx(self::loadBuiltins($user, array($name)), $name);
  }


/* -(  PhabricatorPolicyInterface Implementation  )-------------------------- */


  public function getCapabilities() {
    return array(
      PhabricatorPolicyCapability::CAN_VIEW,
    );
  }

  public function getPolicy($capability) {
    // TODO: Implement proper per-object policies.
    return PhabricatorPolicies::POLICY_USER;
  }

  public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
    return false;
  }

  public function describeAutomaticCapability($capability) {
    return null;
  }


/* -(  PhabricatorSubscribableInterface Implementation  )-------------------- */


  public function isAutomaticallySubscribed($phid) {
    return ($this->authorPHID == $phid);
  }


/* -(  PhabricatorTokenReceiverInterface  )---------------------------------- */


  public function getUsersToNotifyOfTokenGiven() {
    return array(
      $this->getAuthorPHID(),
    );
  }


}