Swap S3 to first-party client

Summary:
Ref T5155. Swaps Phabricator over to the new first-party S3 client using the v4 authentication API so it works in all regions.

The API requires an explicit region, so the new `amazon-s3.region` is now required. I'll write guidance about this.

Test Plan:
  - Uploaded files to S3.
  - Migrated ~1GB of files to S3.
  - Loaded a bunch of files off S3.
  - Browsed around the S3 bucket.
  - Deleted a file, verified the data on S3 was destroyed.
  - Hit new setup warning.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T5155

Differential Revision: https://secure.phabricator.com/D14982

externals/s3/README.txt

@@ -1,105 +0,0 @@
-AMAZON S3 PHP CLASS
-
-
-USING THE CLASS
-
-OO method (e,g; $s3->getObject(...)):
-$s3 = new S3(awsAccessKey, awsSecretKey);
-
-Statically (e,g; S3::getObject(...)):
-S3::setAuth(awsAccessKey, awsSecretKey);
-
-
-For class documentation see:
-http://undesigned.org.za/files/s3-class-documentation/index.html
-
-
-OBJECTS
-
-
-Put an object from a string:
-	$s3->putObject($string, $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
-	Legacy function: $s3->putObjectString($string, $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
-
-
-Put an object from a file:
-	$s3->putObject($s3->inputFile($file, false), $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
-	Legacy function: $s3->putObjectFile($uploadFile, $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
-
-
-Put an object from a resource (buffer/file size is required):
-	Please note: the resource will be fclose()'d automatically
-	$s3->putObject($s3->inputResource(fopen($file, 'rb'), filesize($file)), $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
-
-
-Get an object:
-	$s3->getObject($bucketName, $uploadName)
-
-
-Save an object to file:
-	$s3->getObject($bucketName, $uploadName, $saveName)
-
-
-Save an object to a resource of any type:
-	$s3->getObject($bucketName, $uploadName, fopen('savefile.txt', 'wb'))
-
-
-Copy an object:
-	$s3->copyObject($srcBucket, $srcName, $bucketName, $saveName, $metaHeaders = array(), $requestHeaders = array())
-
-
-Delete an object:
-	$s3->deleteObject($bucketName, $uploadName)
-
-
-
-BUCKETS
-
-
-Get a list of buckets:
-	$s3->listBuckets()  // Simple bucket list
-	$s3->listBuckets(true)  // Detailed bucket list
-
-
-Create a public-read bucket:
-	$s3->putBucket($bucketName, S3::ACL_PUBLIC_READ)
-	$s3->putBucket($bucketName, S3::ACL_PUBLIC_READ, 'EU') // EU-hosted bucket
-
-
-Get the contents of a bucket:
-	$s3->getBucket($bucketName)
-
-
-Get a bucket's location:
-	$s3->getBucketLocation($bucketName)
-
-
-Delete a bucket:
-	$s3->deleteBucket($bucketName)
-
-
-
-
-KNOWN ISSUES
-
-	Files larger than 2GB are not supported on 32 bit systems due to PHP’s signed integer problem
-
-
-
-MORE INFORMATION
-
-
-	Project URL:
-	http://undesigned.org.za/2007/10/22/amazon-s3-php-class
-
-	Class documentation:
-	http://undesigned.org.za/files/s3-class-documentation/index.html
-
-	Bug reports:
-	https://github.com/tpyo/amazon-s3-php-class/issues
-
-	Amazon S3 documentation:
-	http://docs.amazonwebservices.com/AmazonS3/2006-03-01/
-
-
-EOF

src/applications/config/check/PhabricatorStorageSetupCheck.php

@@ -13,6 +13,8 @@ final class PhabricatorStorageSetupCheck extends PhabricatorSetupCheck {
     $engines = PhabricatorFileStorageEngine::loadWritableChunkEngines();
     $chunk_engine_active = (bool)$engines;
 
+    $this->checkS3();
+
     if (!$chunk_engine_active) {
       $doc_href = PhabricatorEnv::getDocLink('Configuring File Storage');
 
@@ -140,4 +142,55 @@ final class PhabricatorStorageSetupCheck extends PhabricatorSetupCheck {
         ->addPhabricatorConfig('storage.local-disk.path');
     }
   }
+
+  private function checkS3() {
+    $access_key = PhabricatorEnv::getEnvConfig('amazon-s3.access-key');
+    $secret_key = PhabricatorEnv::getEnvConfig('amazon-s3.secret-key');
+    $region = PhabricatorEnv::getEnvConfig('amazon-s3.region');
+    $endpoint = PhabricatorEnv::getEnvConfig('amazon-s3.endpoint');
+
+    $how_many = 0;
+
+    if (strlen($access_key)) {
+      $how_many++;
+    }
+
+    if (strlen($secret_key)) {
+      $how_many++;
+    }
+
+    if (strlen($region)) {
+      $how_many++;
+    }
+
+    if (strlen($endpoint)) {
+      $how_many++;
+    }
+
+    // Nothing configured, no issues here.
+    if ($how_many === 0) {
+      return;
+    }
+
+    // Everything configured, no issues here.
+    if ($how_many === 4) {
+      return;
+    }
+
+    $message = pht(
+      'File storage in Amazon S3 has been partially configured, but you are '.
+      'missing some required settings. S3 will not be available to store '.
+      'files until you complete the configuration. Either configure S3 fully '.
+      'or remove the partial configuration.');
+
+    $this->newIssue('storage.s3.partial-config')
+      ->setShortName(pht('S3 Partially Configured'))
+      ->setName(pht('Amazon S3 is Only Partially Configured'))
+      ->setMessage($message)
+      ->addPhabricatorConfig('amazon-s3.access-key')
+      ->addPhabricatorConfig('amazon-s3.secret-key')
+      ->addPhabricatorConfig('amazon-s3.region')
+      ->addPhabricatorConfig('amazon-s3.endpoint');
+  }
+
 }

src/applications/config/option/PhabricatorAWSConfigOptions.php

@@ -33,14 +33,27 @@ final class PhabricatorAWSConfigOptions
       $this->newOption('amazon-s3.secret-key', 'string', null)
         ->setHidden(true)
         ->setDescription(pht('Secret key for Amazon S3.')),
+      $this->newOption('amazon-s3.region', 'string', null)
+        ->setLocked(true)
+        ->setDescription(
+          pht(
+            'Amazon S3 region where your S3 bucket is located. When you '.
+            'specify a region, you should also specify a corresponding '.
+            'endpoint with `amazon-s3.endpoint`. You can find a list of '.
+            'available regions and endpoints in the AWS documentation.'))
+        ->addExample('us-west-1', pht('USWest Region')),
       $this->newOption('amazon-s3.endpoint', 'string', null)
         ->setLocked(true)
         ->setDescription(
           pht(
-            'Explicit S3 endpoint to use. Leave empty to have Phabricator '.
+            'Explicit S3 endpoint to use. This should be the endpoint '.
-            'select and endpoint. Normally, you do not need to set this.'))
+            'which corresponds to the region you have selected in '.
-        ->addExample(null, pht('Use default endpoint'))
+            '`amazon-s3.region`. Phabricator can not determine the correct '.
-        ->addExample('s3.amazon.com', pht('Use specific endpoint')),
+            'endpoint automatically because some endpoint locations are '.
+            'irregular.'))
+        ->addExample(
+          's3-us-west-1.amazonaws.com',
+          pht('Use specific endpoint')),
       $this->newOption('amazon-ec2.access-key', 'string', null)
         ->setLocked(true)
         ->setDescription(pht('Access key for Amazon EC2.')),

src/applications/config/view/PhabricatorSetupIssueView.php

@@ -275,20 +275,21 @@ final class PhabricatorSetupIssueView extends AphrontView {
       $update = array();
       foreach ($configs as $config) {
         if (idx($options, $config) && $options[$config]->getLocked()) {
-          continue;
+          $name = pht('View "%s"', $config);
+        } else {
+          $name = pht('Edit "%s"', $config);
         }
         $link = phutil_tag(
           'a',
           array(
             'href' => '/config/edit/'.$config.'/?issue='.$issue->getIssueKey(),
           ),
-          pht('Edit %s', $config));
+          $name);
         $update[] = phutil_tag('li', array(), $link);
       }
       if ($update) {
         $update = phutil_tag('ul', array(), $update);
         if (!$related) {
-
           $update_info = phutil_tag(
           'p',
           array(),

src/applications/files/engine/PhabricatorS3FileStorageEngine.php

@@ -28,8 +28,14 @@ final class PhabricatorS3FileStorageEngine
     $bucket = PhabricatorEnv::getEnvConfig('storage.s3.bucket');
     $access_key = PhabricatorEnv::getEnvConfig('amazon-s3.access-key');
     $secret_key = PhabricatorEnv::getEnvConfig('amazon-s3.secret-key');
+    $endpoint = PhabricatorEnv::getEnvConfig('amazon-s3.endpoint');
+    $region = PhabricatorEnv::getEnvConfig('amazon-s3.region');
 
-    return (strlen($bucket) && strlen($access_key) && strlen($secret_key));
+    return (strlen($bucket) &&
+      strlen($access_key) &&
+      strlen($secret_key) &&
+      strlen($endpoint) &&
+      strlen($region));
   }
 
 
@@ -68,11 +74,11 @@ final class PhabricatorS3FileStorageEngine
         'type' => 's3',
         'method' => 'putObject',
       ));
-    $s3->putObject(
+
-      $data,
+    $s3
-      $this->getBucketName(),
+      ->setParametersForPutObject($name, $data)
-      $name,
+      ->resolve();
-      $acl = 'private');
+
     $profiler->endServiceCall($call_id, array());
 
     return $name;
@@ -84,24 +90,21 @@ final class PhabricatorS3FileStorageEngine
    */
   public function readFile($handle) {
     $s3 = $this->newS3API();
+
     $profiler = PhutilServiceProfiler::getInstance();
     $call_id = $profiler->beginServiceCall(
       array(
         'type' => 's3',
         'method' => 'getObject',
       ));
-    $result = $s3->getObject(
+
-      $this->getBucketName(),
+    $result = $s3
-      $handle);
+      ->setParametersForGetObject($handle)
+      ->resolve();
+
     $profiler->endServiceCall($call_id, array());
 
-    // NOTE: The implementation of the API that we're using may respond with
+    return $result;
-    // a successful result that has length 0 and no body property.
-    if (isset($result->body)) {
-      return $result->body;
-    } else {
-      return '';
-    }
   }
 
 
@@ -109,17 +112,20 @@ final class PhabricatorS3FileStorageEngine
    * Delete a blob from Amazon S3.
    */
   public function deleteFile($handle) {
-    AphrontWriteGuard::willWrite();
     $s3 = $this->newS3API();
+
+    AphrontWriteGuard::willWrite();
     $profiler = PhutilServiceProfiler::getInstance();
     $call_id = $profiler->beginServiceCall(
       array(
         'type' => 's3',
         'method' => 'deleteObject',
       ));
-    $s3->deleteObject(
+
-      $this->getBucketName(),
+    $s3
-      $handle);
+      ->setParametersForDeleteObject($handle)
+      ->resolve();
+
     $profiler->endServiceCall($call_id, array());
   }
 
@@ -147,33 +153,19 @@ final class PhabricatorS3FileStorageEngine
    * Create a new S3 API object.
    *
    * @task internal
-   * @phutil-external-symbol class S3
    */
   private function newS3API() {
-    $libroot = dirname(phutil_get_library_root('phabricator'));
-    require_once $libroot.'/externals/s3/S3.php';
-
     $access_key = PhabricatorEnv::getEnvConfig('amazon-s3.access-key');
     $secret_key = PhabricatorEnv::getEnvConfig('amazon-s3.secret-key');
+    $region = PhabricatorEnv::getEnvConfig('amazon-s3.region');
     $endpoint = PhabricatorEnv::getEnvConfig('amazon-s3.endpoint');
 
-    if (!$access_key || !$secret_key) {
+    return id(new PhutilAWSS3Future())
-      throw new PhabricatorFileStorageConfigurationException(
+      ->setAccessKey($access_key)
-        pht(
+      ->setSecretKey(new PhutilOpaqueEnvelope($secret_key))
-          "Specify '%s' and '%s'!",
+      ->setRegion($region)
-          'amazon-s3.access-key',
+      ->setEndpoint($endpoint)
-          'amazon-s3.secret-key'));
+      ->setBucket($this->getBucketName());
-    }
-
-    if ($endpoint !== null) {
-      $s3 = new S3($access_key, $secret_key, $use_ssl = true, $endpoint);
-    } else {
-      $s3 = new S3($access_key, $secret_key, $use_ssl = true);
-    }
-
-    $s3->setExceptions(true);
-
-    return $s3;
   }
 
 }

src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php

@@ -18,6 +18,20 @@ final class PhabricatorFilesManagementMigrateWorkflow
             'name'      => 'dry-run',
             'help'      => pht('Show what would be migrated.'),
           ),
+          array(
+            'name' => 'min-size',
+            'param' => 'bytes',
+            'help' => pht(
+              'Do not migrate data for files which are smaller than a given '.
+              'filesize.'),
+          ),
+          array(
+            'name' => 'max-size',
+            'param' => 'bytes',
+            'help' => pht(
+              'Do not migrate data for files which are larger than a given '.
+              'filesize.'),
+          ),
           array(
             'name'      => 'all',
             'help'      => pht('Migrate all files.'),
@@ -53,8 +67,13 @@ final class PhabricatorFilesManagementMigrateWorkflow
 
     $is_dry_run = $args->getArg('dry-run');
 
+    $min_size = (int)$args->getArg('min-size');
+    $max_size = (int)$args->getArg('max-size');
+
     $failed = array();
     $engines = PhabricatorFileStorageEngine::loadAllEngines();
+    $total_bytes = 0;
+    $total_files = 0;
     foreach ($iterator as $file) {
       $monogram = $file->getMonogram();
 
@@ -91,27 +110,59 @@ final class PhabricatorFilesManagementMigrateWorkflow
         continue;
       }
 
+      $byte_size = $file->getByteSize();
+
+      if ($min_size && ($byte_size < $min_size)) {
+        echo tsprintf(
+          "%s\n",
+          pht(
+            '%s: File size (%s) is smaller than minimum size (%s).',
+            $monogram,
+            phutil_format_bytes($byte_size),
+            phutil_format_bytes($min_size)));
+        continue;
+      }
+
+      if ($max_size && ($byte_size > $max_size)) {
+        echo tsprintf(
+          "%s\n",
+          pht(
+            '%s: File size (%s) is larger than maximum size (%s).',
+            $monogram,
+            phutil_format_bytes($byte_size),
+            phutil_format_bytes($max_size)));
+        continue;
+      }
+
       if ($is_dry_run) {
         echo tsprintf(
           "%s\n",
           pht(
-            '%s: Would migrate from "%s" to "%s" (dry run).',
+            '%s: (%s) Would migrate from "%s" to "%s" (dry run)...',
             $monogram,
+            phutil_format_bytes($byte_size),
+            $engine_key,
+            $target_key));
+      } else {
+        echo tsprintf(
+          "%s\n",
+          pht(
+            '%s: (%s) Migrating from "%s" to "%s"...',
+            $monogram,
+            phutil_format_bytes($byte_size),
             $engine_key,
             $target_key));
-        continue;
       }
 
-      echo tsprintf(
-        "%s\n",
-        pht(
-          '%s: Migrating from "%s" to "%s"...',
-          $monogram,
-          $engine_key,
-          $target_key));
-
       try {
-        $file->migrateToEngine($target_engine);
+        if ($is_dry_run) {
+          // Do nothing, this is a dry run.
+        } else {
+          $file->migrateToEngine($target_engine);
+        }
+
+        $total_files += 1;
+        $total_bytes += $byte_size;
 
         echo tsprintf(
           "%s\n",
@@ -127,6 +178,25 @@ final class PhabricatorFilesManagementMigrateWorkflow
       }
     }
 
+    echo tsprintf(
+      "%s\n",
+      pht(
+        'Total Migrated Files: %s',
+        new PhutilNumber($total_files)));
+
+    echo tsprintf(
+      "%s\n",
+      pht(
+        'Total Migrated Bytes: %s',
+        phutil_format_bytes($total_bytes)));
+
+    if ($is_dry_run) {
+      echo tsprintf(
+        "%s\n",
+        pht(
+          'This was a dry run, so no real migrations were performed.'));
+    }
+
     if ($failed) {
       $monograms = mpull($failed, 'getMonogram');