Add an Amazon S3 storage engine for Phabricator

Summary: Implements an S3 storage engine option for Phabricator.
Test Plan:
  - Uploaded files to S3.
  - Looked at them.
  - Verified they appeared in S3 using the S3 file browser.

Reviewed By: jungejason
Reviewers: jungejason, tuomaspelkonen, aran
CC: aran, jungejason
Differential Revision: 752

conf/default.conf.php

@@ -377,7 +377,14 @@ return array(
   // disable it, specify null.
   'storage.local-disk.path'       => null,
 
-  // TODO: Implement S3.
+  // If you want to store files in Amazon S3, specify an AWS access and secret
+  // key here and a bucket name below.
+  'amazon-s3.access-key'          =>  null,
+  'amazon-s3.secret-key'          =>  null,
+
+  // Set this to a valid Amazon S3 bucket to store files there. You must also
+  // configure S3 access keys above.
+  'storage.s3.bucket'             => null,
 
   // Phabricator uses a storage engine selector to choose which storage engine
   // to use when writing file data. If you add new storage engines or want to

externals/s3/README.txt

@@ -0,0 +1,105 @@
+AMAZON S3 PHP CLASS
+
+
+USING THE CLASS
+
+OO method (e,g; $s3->getObject(...)):
+$s3 = new S3(awsAccessKey, awsSecretKey);
+
+Statically (e,g; S3::getObject(...)):
+S3::setAuth(awsAccessKey, awsSecretKey);
+
+
+For class documentation see:
+http://undesigned.org.za/files/s3-class-documentation/index.html
+
+
+OBJECTS
+
+
+Put an object from a string:
+	$s3->putObject($string, $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
+	Legacy function: $s3->putObjectString($string, $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
+
+
+Put an object from a file:
+	$s3->putObject($s3->inputFile($file, false), $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
+	Legacy function: $s3->putObjectFile($uploadFile, $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
+
+
+Put an object from a resource (buffer/file size is required):
+	Please note: the resource will be fclose()'d automatically
+	$s3->putObject($s3->inputResource(fopen($file, 'rb'), filesize($file)), $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
+
+
+Get an object:
+	$s3->getObject($bucketName, $uploadName)
+
+
+Save an object to file:
+	$s3->getObject($bucketName, $uploadName, $saveName)
+
+
+Save an object to a resource of any type:
+	$s3->getObject($bucketName, $uploadName, fopen('savefile.txt', 'wb'))
+
+
+Copy an object:
+	$s3->copyObject($srcBucket, $srcName, $bucketName, $saveName, $metaHeaders = array(), $requestHeaders = array())
+
+
+Delete an object:
+	$s3->deleteObject($bucketName, $uploadName)
+
+
+
+BUCKETS
+
+
+Get a list of buckets:
+	$s3->listBuckets()  // Simple bucket list
+	$s3->listBuckets(true)  // Detailed bucket list
+
+
+Create a public-read bucket:
+	$s3->putBucket($bucketName, S3::ACL_PUBLIC_READ)
+	$s3->putBucket($bucketName, S3::ACL_PUBLIC_READ, 'EU') // EU-hosted bucket
+
+
+Get the contents of a bucket:
+	$s3->getBucket($bucketName)
+
+
+Get a bucket's location:
+	$s3->getBucketLocation($bucketName)
+
+
+Delete a bucket:
+	$s3->deleteBucket($bucketName)
+
+
+
+
+KNOWN ISSUES
+
+	Files larger than 2GB are not supported on 32 bit systems due to PHP’s signed integer problem
+
+
+
+MORE INFORMATION
+
+
+	Project URL:
+	http://undesigned.org.za/2007/10/22/amazon-s3-php-class
+
+	Class documentation:
+	http://undesigned.org.za/files/s3-class-documentation/index.html
+
+	Bug reports:
+	https://github.com/tpyo/amazon-s3-php-class/issues
+
+	Amazon S3 documentation:
+	http://docs.amazonwebservices.com/AmazonS3/2006-03-01/
+
+
+EOF

src/__phutil_library_map__.php

@@ -523,6 +523,7 @@ phutil_register_library_map(array(
     'PhabricatorRepositorySvnCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/svn',
     'PhabricatorRepositorySvnCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/svn',
     'PhabricatorRepositoryType' => 'applications/repository/constants/repositorytype',
+    'PhabricatorS3FileStorageEngine' => 'applications/files/engine/s3',
     'PhabricatorSQLPatchList' => 'infrastructure/setup/sql',
     'PhabricatorSearchAbstractDocument' => 'applications/search/index/abstractdocument',
     'PhabricatorSearchAttachController' => 'applications/search/controller/attach',
@@ -1061,6 +1062,7 @@ phutil_register_library_map(array(
     'PhabricatorRepositorySvnCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
     'PhabricatorRepositorySvnCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
     'PhabricatorRepositorySvnCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
+    'PhabricatorS3FileStorageEngine' => 'PhabricatorFileStorageEngine',
     'PhabricatorSearchAttachController' => 'PhabricatorSearchController',
     'PhabricatorSearchBaseController' => 'PhabricatorController',
     'PhabricatorSearchCommitIndexer' => 'PhabricatorSearchDocumentIndexer',

src/applications/files/engine/s3/PhabricatorS3FileStorageEngine.php

@@ -0,0 +1,131 @@
+<?php
+
+/*
+ * Copyright 2011 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Amazon S3 file storage engine. This engine scales well but is relatively
+ * high-latency since data has to be pulled off S3.
+ *
+ * @task impl       Implementation
+ * @task internal   Internals
+ * @group filestorage
+ */
+final class PhabricatorS3FileStorageEngine
+  extends PhabricatorFileStorageEngine {
+
+/* -(  Implementation  )----------------------------------------------------- */
+
+
+  /**
+   * This engine identifies as "amazon-s3".
+   *
+   * @task impl
+   */
+  public function getEngineIdentifier() {
+    return 'amazon-s3';
+  }
+
+
+  /**
+   * Write file data into S3.
+   * @task impl
+   */
+  public function writeFile($data, array $params) {
+    $s3 = $this->newS3API();
+
+    $name = 'phabricator/'.sha1(Filesystem::readRandomBytes(20));
+
+    $s3->putObject(
+      $data,
+      $this->getBucketName(),
+      $name,
+      $acl = 'private');
+
+    return $name;
+  }
+
+
+  /**
+   * Load a stored blob from S3.
+   * @task impl
+   */
+  public function readFile($handle) {
+    $result = $this->newS3API()->getObject(
+      $this->getBucketName(),
+      $handle);
+    return $result->body;
+  }
+
+
+  /**
+   * Delete a blob from S3.
+   * @task impl
+   */
+  public function deleteFile($handle) {
+    $this->newS3API()->deleteObject(
+      $this->getBucketName(),
+      $handle);
+  }
+
+
+/* -(  Internals  )---------------------------------------------------------- */
+
+
+  /**
+   * Retrieve the S3 bucket name.
+   *
+   * @task internal
+   */
+  private function getBucketName() {
+    $bucket = PhabricatorEnv::getEnvConfig('storage.s3.bucket');
+    if (!$bucket) {
+      throw new Exception("No 'storage.s3.bucket' specified!");
+    }
+    return $bucket;
+  }
+
+  /**
+   * Create a new S3 API object.
+   *
+   * @task internal
+   */
+  private function newS3API() {
+    $libroot = dirname(phutil_get_library_root('phabricator'));
+    require_once $libroot.'/externals/s3/S3.php';
+
+    $access_key = PhabricatorEnv::getEnvConfig('amazon-s3.access-key');
+    $secret_key = PhabricatorEnv::getEnvConfig('amazon-s3.secret-key');
+
+    if (!$access_key || !$secret_key) {
+      throw new Exception(
+        "Specify 'amazon-s3.access-key' and 'amazon-s3.secret-key'!");
+    }
+
+    $s3 = newv(
+      'S3',
+      array(
+        $access_key,
+        $secret_key,
+        $use_ssl = true,
+      ));
+
+    $s3->setExceptions(true);
+
+    return $s3;
+  }
+
+}

src/applications/files/engine/s3/__init__.php

@@ -0,0 +1,17 @@
+<?php
+/**
+ * This file is automatically generated. Lint this module to rebuild it.
+ * @generated
+ */
+
+
+
+phutil_require_module('phabricator', 'applications/files/engine/base');
+phutil_require_module('phabricator', 'infrastructure/env');
+
+phutil_require_module('phutil', 'filesystem');
+phutil_require_module('phutil', 'moduleutils');
+phutil_require_module('phutil', 'utils');
+
+
+phutil_require_source('PhabricatorS3FileStorageEngine.php');

src/applications/files/engineselector/default/PhabricatorDefaultFileStorageEngineSelector.php

@@ -48,6 +48,11 @@ final class PhabricatorDefaultFileStorageEngineSelector
       $engines[] = new PhabricatorLocalDiskFileStorageEngine();
     }
 
+    $s3_key = 'storage.s3.bucket';
+    if (PhabricatorEnv::getEnvConfig($s3_key)) {
+      $engines[] = new PhabricatorS3FileStorageEngine();
+    }
+
     if ($mysql_limit && empty($engines)) {
       // If we return no engines, an exception will be thrown but it will be
       // a little vague ("No valid storage engines"). Since this is a default

src/applications/files/engineselector/default/__init__.php

@@ -8,6 +8,7 @@
 
 phutil_require_module('phabricator', 'applications/files/engine/localdisk');
 phutil_require_module('phabricator', 'applications/files/engine/mysql');
+phutil_require_module('phabricator', 'applications/files/engine/s3');
 phutil_require_module('phabricator', 'applications/files/engineselector/base');
 phutil_require_module('phabricator', 'infrastructure/env');