From cb087570327832c536a556e9833e9a64c8479b35 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sat, 9 Jan 2016 12:43:04 -0800 Subject: [PATCH] Swap S3 to first-party client Summary: Ref T5155. Swaps Phabricator over to the new first-party S3 client using the v4 authentication API so it works in all regions. The API requires an explicit region, so the new `amazon-s3.region` is now required. I'll write guidance about this. Test Plan: - Uploaded files to S3. - Migrated ~1GB of files to S3. - Loaded a bunch of files off S3. - Browsed around the S3 bucket. - Deleted a file, verified the data on S3 was destroyed. - Hit new setup warning. Reviewers: chad Reviewed By: chad Maniphest Tasks: T5155 Differential Revision: https://secure.phabricator.com/D14982 --- externals/s3/README.txt | 105 - externals/s3/S3.php | 2317 ----------------- .../check/PhabricatorStorageSetupCheck.php | 53 + .../option/PhabricatorAWSConfigOptions.php | 21 +- .../config/view/PhabricatorSetupIssueView.php | 7 +- .../engine/PhabricatorS3FileStorageEngine.php | 74 +- ...bricatorFilesManagementMigrateWorkflow.php | 92 +- 7 files changed, 188 insertions(+), 2481 deletions(-) delete mode 100644 externals/s3/README.txt delete mode 100644 externals/s3/S3.php diff --git a/externals/s3/README.txt b/externals/s3/README.txt deleted file mode 100644 index 28a9e73f92..0000000000 --- a/externals/s3/README.txt +++ /dev/null @@ -1,105 +0,0 @@ -AMAZON S3 PHP CLASS - - -USING THE CLASS - -OO method (e,g; $s3->getObject(...)): -$s3 = new S3(awsAccessKey, awsSecretKey); - -Statically (e,g; S3::getObject(...)): -S3::setAuth(awsAccessKey, awsSecretKey); - - -For class documentation see: -http://undesigned.org.za/files/s3-class-documentation/index.html - - -OBJECTS - - -Put an object from a string: - $s3->putObject($string, $bucketName, $uploadName, S3::ACL_PUBLIC_READ) - Legacy function: $s3->putObjectString($string, $bucketName, $uploadName, S3::ACL_PUBLIC_READ) - - -Put an object from a file: - $s3->putObject($s3->inputFile($file, false), $bucketName, $uploadName, S3::ACL_PUBLIC_READ) - Legacy function: $s3->putObjectFile($uploadFile, $bucketName, $uploadName, S3::ACL_PUBLIC_READ) - - -Put an object from a resource (buffer/file size is required): - Please note: the resource will be fclose()'d automatically - $s3->putObject($s3->inputResource(fopen($file, 'rb'), filesize($file)), $bucketName, $uploadName, S3::ACL_PUBLIC_READ) - - -Get an object: - $s3->getObject($bucketName, $uploadName) - - -Save an object to file: - $s3->getObject($bucketName, $uploadName, $saveName) - - -Save an object to a resource of any type: - $s3->getObject($bucketName, $uploadName, fopen('savefile.txt', 'wb')) - - -Copy an object: - $s3->copyObject($srcBucket, $srcName, $bucketName, $saveName, $metaHeaders = array(), $requestHeaders = array()) - - -Delete an object: - $s3->deleteObject($bucketName, $uploadName) - - - -BUCKETS - - -Get a list of buckets: - $s3->listBuckets() // Simple bucket list - $s3->listBuckets(true) // Detailed bucket list - - -Create a public-read bucket: - $s3->putBucket($bucketName, S3::ACL_PUBLIC_READ) - $s3->putBucket($bucketName, S3::ACL_PUBLIC_READ, 'EU') // EU-hosted bucket - - -Get the contents of a bucket: - $s3->getBucket($bucketName) - - -Get a bucket's location: - $s3->getBucketLocation($bucketName) - - -Delete a bucket: - $s3->deleteBucket($bucketName) - - - - -KNOWN ISSUES - - Files larger than 2GB are not supported on 32 bit systems due to PHP’s signed integer problem - - - -MORE INFORMATION - - - Project URL: - http://undesigned.org.za/2007/10/22/amazon-s3-php-class - - Class documentation: - http://undesigned.org.za/files/s3-class-documentation/index.html - - Bug reports: - https://github.com/tpyo/amazon-s3-php-class/issues - - Amazon S3 documentation: - http://docs.amazonwebservices.com/AmazonS3/2006-03-01/ - - -EOF diff --git a/externals/s3/S3.php b/externals/s3/S3.php deleted file mode 100644 index 70e305e43f..0000000000 --- a/externals/s3/S3.php +++ /dev/null @@ -1,2317 +0,0 @@ - $host, 'type' => $type, 'user' => $user, 'pass' => $pass); - } - - - /** - * Set the error mode to exceptions - * - * @param boolean $enabled Enable exceptions - * @return void - */ - public static function setExceptions($enabled = true) - { - self::$useExceptions = $enabled; - } - - - /** - * Set signing key - * - * @param string $keyPairId AWS Key Pair ID - * @param string $signingKey Private Key - * @param boolean $isFile Load private key from file, set to false to load string - * @return boolean - */ - public static function setSigningKey($keyPairId, $signingKey, $isFile = true) - { - self::$__signingKeyPairId = $keyPairId; - if ((self::$__signingKeyResource = openssl_pkey_get_private($isFile ? - file_get_contents($signingKey) : $signingKey)) !== false) return true; - self::__triggerError('S3::setSigningKey(): Unable to open load private key: '.$signingKey, __FILE__, __LINE__); - return false; - } - - - /** - * Free signing key from memory, MUST be called if you are using setSigningKey() - * - * @return void - */ - public static function freeSigningKey() - { - if (self::$__signingKeyResource !== false) - openssl_free_key(self::$__signingKeyResource); - } - - - /** - * Internal error handler - * - * @internal Internal error handler - * @param string $message Error message - * @param string $file Filename - * @param integer $line Line number - * @param integer $code Error code - * @return void - */ - private static function __triggerError($message, $file, $line, $code = 0) - { - if (self::$useExceptions) - throw new S3Exception($message, $file, $line, $code); - else - trigger_error($message, E_USER_WARNING); - } - - - /** - * Get a list of buckets - * - * @param boolean $detailed Returns detailed bucket list when true - * @return array | false - */ - public static function listBuckets($detailed = false) - { - $rest = new S3Request('GET', '', '', self::$endpoint); - $rest = $rest->getResponse(); - if ($rest->error === false && $rest->code !== 200) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - self::__triggerError(sprintf("S3::listBuckets(): [%s] %s", $rest->error['code'], - $rest->error['message']), __FILE__, __LINE__); - return false; - } - $results = array(); - if (!isset($rest->body->Buckets)) return $results; - - if ($detailed) - { - if (isset($rest->body->Owner, $rest->body->Owner->ID, $rest->body->Owner->DisplayName)) - $results['owner'] = array( - 'id' => (string)$rest->body->Owner->ID, 'name' => (string)$rest->body->Owner->ID - ); - $results['buckets'] = array(); - foreach ($rest->body->Buckets->Bucket as $b) - $results['buckets'][] = array( - 'name' => (string)$b->Name, 'time' => strtotime((string)$b->CreationDate) - ); - } else - foreach ($rest->body->Buckets->Bucket as $b) $results[] = (string)$b->Name; - - return $results; - } - - - /** - * Get contents for a bucket - * - * If maxKeys is null this method will loop through truncated result sets - * - * @param string $bucket Bucket name - * @param string $prefix Prefix - * @param string $marker Marker (last file listed) - * @param string $maxKeys Max keys (maximum number of keys to return) - * @param string $delimiter Delimiter - * @param boolean $returnCommonPrefixes Set to true to return CommonPrefixes - * @return array | false - */ - public static function getBucket($bucket, $prefix = null, $marker = null, $maxKeys = null, $delimiter = null, $returnCommonPrefixes = false) - { - $rest = new S3Request('GET', $bucket, '', self::$endpoint); - if ($maxKeys == 0) $maxKeys = null; - if ($prefix !== null && $prefix !== '') $rest->setParameter('prefix', $prefix); - if ($marker !== null && $marker !== '') $rest->setParameter('marker', $marker); - if ($maxKeys !== null && $maxKeys !== '') $rest->setParameter('max-keys', $maxKeys); - if ($delimiter !== null && $delimiter !== '') $rest->setParameter('delimiter', $delimiter); - $response = $rest->getResponse(); - if ($response->error === false && $response->code !== 200) - $response->error = array('code' => $response->code, 'message' => 'Unexpected HTTP status'); - if ($response->error !== false) - { - self::__triggerError(sprintf("S3::getBucket(): [%s] %s", - $response->error['code'], $response->error['message']), __FILE__, __LINE__); - return false; - } - - $results = array(); - - $nextMarker = null; - if (isset($response->body, $response->body->Contents)) - foreach ($response->body->Contents as $c) - { - $results[(string)$c->Key] = array( - 'name' => (string)$c->Key, - 'time' => strtotime((string)$c->LastModified), - 'size' => (int)$c->Size, - 'hash' => substr((string)$c->ETag, 1, -1) - ); - $nextMarker = (string)$c->Key; - } - - if ($returnCommonPrefixes && isset($response->body, $response->body->CommonPrefixes)) - foreach ($response->body->CommonPrefixes as $c) - $results[(string)$c->Prefix] = array('prefix' => (string)$c->Prefix); - - if (isset($response->body, $response->body->IsTruncated) && - (string)$response->body->IsTruncated == 'false') return $results; - - if (isset($response->body, $response->body->NextMarker)) - $nextMarker = (string)$response->body->NextMarker; - - // Loop through truncated results if maxKeys isn't specified - if ($maxKeys == null && $nextMarker !== null && (string)$response->body->IsTruncated == 'true') - do - { - $rest = new S3Request('GET', $bucket, '', self::$endpoint); - if ($prefix !== null && $prefix !== '') $rest->setParameter('prefix', $prefix); - $rest->setParameter('marker', $nextMarker); - if ($delimiter !== null && $delimiter !== '') $rest->setParameter('delimiter', $delimiter); - - if (($response = $rest->getResponse()) == false || $response->code !== 200) break; - - if (isset($response->body, $response->body->Contents)) - foreach ($response->body->Contents as $c) - { - $results[(string)$c->Key] = array( - 'name' => (string)$c->Key, - 'time' => strtotime((string)$c->LastModified), - 'size' => (int)$c->Size, - 'hash' => substr((string)$c->ETag, 1, -1) - ); - $nextMarker = (string)$c->Key; - } - - if ($returnCommonPrefixes && isset($response->body, $response->body->CommonPrefixes)) - foreach ($response->body->CommonPrefixes as $c) - $results[(string)$c->Prefix] = array('prefix' => (string)$c->Prefix); - - if (isset($response->body, $response->body->NextMarker)) - $nextMarker = (string)$response->body->NextMarker; - - } while ($response !== false && (string)$response->body->IsTruncated == 'true'); - - return $results; - } - - - /** - * Put a bucket - * - * @param string $bucket Bucket name - * @param constant $acl ACL flag - * @param string $location Set as "EU" to create buckets hosted in Europe - * @return boolean - */ - public static function putBucket($bucket, $acl = self::ACL_PRIVATE, $location = false) - { - $rest = new S3Request('PUT', $bucket, '', self::$endpoint); - $rest->setAmzHeader('x-amz-acl', $acl); - - if ($location !== false) - { - $dom = new DOMDocument; - $createBucketConfiguration = $dom->createElement('CreateBucketConfiguration'); - $locationConstraint = $dom->createElement('LocationConstraint', $location); - $createBucketConfiguration->appendChild($locationConstraint); - $dom->appendChild($createBucketConfiguration); - $rest->data = $dom->saveXML(); - $rest->size = strlen($rest->data); - $rest->setHeader('Content-Type', 'application/xml'); - } - $rest = $rest->getResponse(); - - if ($rest->error === false && $rest->code !== 200) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - self::__triggerError(sprintf("S3::putBucket({$bucket}, {$acl}, {$location}): [%s] %s", - $rest->error['code'], $rest->error['message']), __FILE__, __LINE__); - return false; - } - return true; - } - - - /** - * Delete an empty bucket - * - * @param string $bucket Bucket name - * @return boolean - */ - public static function deleteBucket($bucket) - { - $rest = new S3Request('DELETE', $bucket, '', self::$endpoint); - $rest = $rest->getResponse(); - if ($rest->error === false && $rest->code !== 204) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - self::__triggerError(sprintf("S3::deleteBucket({$bucket}): [%s] %s", - $rest->error['code'], $rest->error['message']), __FILE__, __LINE__); - return false; - } - return true; - } - - - /** - * Create input info array for putObject() - * - * @param string $file Input file - * @param mixed $md5sum Use MD5 hash (supply a string if you want to use your own) - * @return array | false - */ - public static function inputFile($file, $md5sum = true) - { - if (!file_exists($file) || !is_file($file) || !is_readable($file)) - { - self::__triggerError('S3::inputFile(): Unable to open input file: '.$file, __FILE__, __LINE__); - return false; - } - return array('file' => $file, 'size' => filesize($file), 'md5sum' => $md5sum !== false ? - (is_string($md5sum) ? $md5sum : base64_encode(md5_file($file, true))) : ''); - } - - - /** - * Create input array info for putObject() with a resource - * - * @param string $resource Input resource to read from - * @param integer $bufferSize Input byte size - * @param string $md5sum MD5 hash to send (optional) - * @return array | false - */ - public static function inputResource(&$resource, $bufferSize = false, $md5sum = '') - { - if (!is_resource($resource) || (int)$bufferSize < 0) - { - self::__triggerError('S3::inputResource(): Invalid resource or buffer size', __FILE__, __LINE__); - return false; - } - - // Try to figure out the bytesize - if ($bufferSize === false) - { - if (fseek($resource, 0, SEEK_END) < 0 || ($bufferSize = ftell($resource)) === false) - { - self::__triggerError('S3::inputResource(): Unable to obtain resource size', __FILE__, __LINE__); - return false; - } - fseek($resource, 0); - } - - $input = array('size' => $bufferSize, 'md5sum' => $md5sum); - $input['fp'] =& $resource; - return $input; - } - - - /** - * Put an object - * - * @param mixed $input Input data - * @param string $bucket Bucket name - * @param string $uri Object URI - * @param constant $acl ACL constant - * @param array $metaHeaders Array of x-amz-meta-* headers - * @param array $requestHeaders Array of request headers or content type as a string - * @param constant $storageClass Storage class constant - * @param constant $serverSideEncryption Server-side encryption - * @return boolean - */ - public static function putObject($input, $bucket, $uri, $acl = self::ACL_PRIVATE, $metaHeaders = array(), $requestHeaders = array(), $storageClass = self::STORAGE_CLASS_STANDARD, $serverSideEncryption = self::SSE_NONE) - { - if ($input === false) return false; - $rest = new S3Request('PUT', $bucket, $uri, self::$endpoint); - - if (!is_array($input)) $input = array( - 'data' => $input, 'size' => strlen($input), - 'md5sum' => base64_encode(md5($input, true)) - ); - - // Data - if (isset($input['fp'])) - $rest->fp =& $input['fp']; - elseif (isset($input['file'])) - $rest->fp = @fopen($input['file'], 'rb'); - elseif (isset($input['data'])) - $rest->data = $input['data']; - - // Content-Length (required) - if (isset($input['size']) && $input['size'] >= 0) - $rest->size = $input['size']; - else { - if (isset($input['file'])) - $rest->size = filesize($input['file']); - elseif (isset($input['data'])) - $rest->size = strlen($input['data']); - } - - // Custom request headers (Content-Type, Content-Disposition, Content-Encoding) - if (is_array($requestHeaders)) - foreach ($requestHeaders as $h => $v) $rest->setHeader($h, $v); - elseif (is_string($requestHeaders)) // Support for legacy contentType parameter - $input['type'] = $requestHeaders; - - // Content-Type - if (!isset($input['type'])) - { - if (isset($requestHeaders['Content-Type'])) - $input['type'] =& $requestHeaders['Content-Type']; - elseif (isset($input['file'])) - $input['type'] = self::__getMimeType($input['file']); - else - $input['type'] = 'application/octet-stream'; - } - - if ($storageClass !== self::STORAGE_CLASS_STANDARD) // Storage class - $rest->setAmzHeader('x-amz-storage-class', $storageClass); - - if ($serverSideEncryption !== self::SSE_NONE) // Server-side encryption - $rest->setAmzHeader('x-amz-server-side-encryption', $serverSideEncryption); - - // We need to post with Content-Length and Content-Type, MD5 is optional - if ($rest->size >= 0 && ($rest->fp !== false || $rest->data !== false)) - { - $rest->setHeader('Content-Type', $input['type']); - if (isset($input['md5sum'])) $rest->setHeader('Content-MD5', $input['md5sum']); - - $rest->setAmzHeader('x-amz-acl', $acl); - foreach ($metaHeaders as $h => $v) $rest->setAmzHeader('x-amz-meta-'.$h, $v); - $rest->getResponse(); - } else - $rest->response->error = array('code' => 0, 'message' => 'Missing input parameters'); - - if ($rest->response->error === false && $rest->response->code !== 200) - $rest->response->error = array('code' => $rest->response->code, 'message' => 'Unexpected HTTP status'); - if ($rest->response->error !== false) - { - self::__triggerError(sprintf("S3::putObject(): [%s] %s", - $rest->response->error['code'], $rest->response->error['message']), __FILE__, __LINE__); - return false; - } - return true; - } - - - /** - * Put an object from a file (legacy function) - * - * @param string $file Input file path - * @param string $bucket Bucket name - * @param string $uri Object URI - * @param constant $acl ACL constant - * @param array $metaHeaders Array of x-amz-meta-* headers - * @param string $contentType Content type - * @return boolean - */ - public static function putObjectFile($file, $bucket, $uri, $acl = self::ACL_PRIVATE, $metaHeaders = array(), $contentType = null) - { - return self::putObject(self::inputFile($file), $bucket, $uri, $acl, $metaHeaders, $contentType); - } - - - /** - * Put an object from a string (legacy function) - * - * @param string $string Input data - * @param string $bucket Bucket name - * @param string $uri Object URI - * @param constant $acl ACL constant - * @param array $metaHeaders Array of x-amz-meta-* headers - * @param string $contentType Content type - * @return boolean - */ - public static function putObjectString($string, $bucket, $uri, $acl = self::ACL_PRIVATE, $metaHeaders = array(), $contentType = 'text/plain') - { - return self::putObject($string, $bucket, $uri, $acl, $metaHeaders, $contentType); - } - - - /** - * Get an object - * - * @param string $bucket Bucket name - * @param string $uri Object URI - * @param mixed $saveTo Filename or resource to write to - * @return mixed - */ - public static function getObject($bucket, $uri, $saveTo = false) - { - $rest = new S3Request('GET', $bucket, $uri, self::$endpoint); - if ($saveTo !== false) - { - if (is_resource($saveTo)) - $rest->fp =& $saveTo; - else - if (($rest->fp = @fopen($saveTo, 'wb')) !== false) - $rest->file = realpath($saveTo); - else - $rest->response->error = array('code' => 0, 'message' => 'Unable to open save file for writing: '.$saveTo); - } - if ($rest->response->error === false) $rest->getResponse(); - - if ($rest->response->error === false && $rest->response->code !== 200) - $rest->response->error = array('code' => $rest->response->code, 'message' => 'Unexpected HTTP status'); - if ($rest->response->error !== false) - { - self::__triggerError(sprintf("S3::getObject({$bucket}, {$uri}): [%s] %s", - $rest->response->error['code'], $rest->response->error['message']), __FILE__, __LINE__); - return false; - } - return $rest->response; - } - - - /** - * Get object information - * - * @param string $bucket Bucket name - * @param string $uri Object URI - * @param boolean $returnInfo Return response information - * @return mixed | false - */ - public static function getObjectInfo($bucket, $uri, $returnInfo = true) - { - $rest = new S3Request('HEAD', $bucket, $uri, self::$endpoint); - $rest = $rest->getResponse(); - if ($rest->error === false && ($rest->code !== 200 && $rest->code !== 404)) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - self::__triggerError(sprintf("S3::getObjectInfo({$bucket}, {$uri}): [%s] %s", - $rest->error['code'], $rest->error['message']), __FILE__, __LINE__); - return false; - } - return $rest->code == 200 ? $returnInfo ? $rest->headers : true : false; - } - - - /** - * Copy an object - * - * @param string $srcBucket Source bucket name - * @param string $srcUri Source object URI - * @param string $bucket Destination bucket name - * @param string $uri Destination object URI - * @param constant $acl ACL constant - * @param array $metaHeaders Optional array of x-amz-meta-* headers - * @param array $requestHeaders Optional array of request headers (content type, disposition, etc.) - * @param constant $storageClass Storage class constant - * @return mixed | false - */ - public static function copyObject($srcBucket, $srcUri, $bucket, $uri, $acl = self::ACL_PRIVATE, $metaHeaders = array(), $requestHeaders = array(), $storageClass = self::STORAGE_CLASS_STANDARD) - { - $rest = new S3Request('PUT', $bucket, $uri, self::$endpoint); - $rest->setHeader('Content-Length', 0); - foreach ($requestHeaders as $h => $v) $rest->setHeader($h, $v); - foreach ($metaHeaders as $h => $v) $rest->setAmzHeader('x-amz-meta-'.$h, $v); - if ($storageClass !== self::STORAGE_CLASS_STANDARD) // Storage class - $rest->setAmzHeader('x-amz-storage-class', $storageClass); - $rest->setAmzHeader('x-amz-acl', $acl); - $rest->setAmzHeader('x-amz-copy-source', sprintf('/%s/%s', $srcBucket, rawurlencode($srcUri))); - if (sizeof($requestHeaders) > 0 || sizeof($metaHeaders) > 0) - $rest->setAmzHeader('x-amz-metadata-directive', 'REPLACE'); - - $rest = $rest->getResponse(); - if ($rest->error === false && $rest->code !== 200) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - self::__triggerError(sprintf("S3::copyObject({$srcBucket}, {$srcUri}, {$bucket}, {$uri}): [%s] %s", - $rest->error['code'], $rest->error['message']), __FILE__, __LINE__); - return false; - } - return isset($rest->body->LastModified, $rest->body->ETag) ? array( - 'time' => strtotime((string)$rest->body->LastModified), - 'hash' => substr((string)$rest->body->ETag, 1, -1) - ) : false; - } - - - /** - * Set up a bucket redirection - * - * @param string $bucket Bucket name - * @param string $location Target host name - * @return boolean - */ - public static function setBucketRedirect($bucket = NULL, $location = NULL) - { - $rest = new S3Request('PUT', $bucket, '', self::$endpoint); - - if( empty($bucket) || empty($location) ) { - self::__triggerError("S3::setBucketRedirect({$bucket}, {$location}): Empty parameter.", __FILE__, __LINE__); - return false; - } - - $dom = new DOMDocument; - $websiteConfiguration = $dom->createElement('WebsiteConfiguration'); - $redirectAllRequestsTo = $dom->createElement('RedirectAllRequestsTo'); - $hostName = $dom->createElement('HostName', $location); - $redirectAllRequestsTo->appendChild($hostName); - $websiteConfiguration->appendChild($redirectAllRequestsTo); - $dom->appendChild($websiteConfiguration); - $rest->setParameter('website', null); - $rest->data = $dom->saveXML(); - $rest->size = strlen($rest->data); - $rest->setHeader('Content-Type', 'application/xml'); - $rest = $rest->getResponse(); - - if ($rest->error === false && $rest->code !== 200) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - self::__triggerError(sprintf("S3::setBucketRedirect({$bucket}, {$location}): [%s] %s", - $rest->error['code'], $rest->error['message']), __FILE__, __LINE__); - return false; - } - return true; - } - - - /** - * Set logging for a bucket - * - * @param string $bucket Bucket name - * @param string $targetBucket Target bucket (where logs are stored) - * @param string $targetPrefix Log prefix (e,g; domain.com-) - * @return boolean - */ - public static function setBucketLogging($bucket, $targetBucket, $targetPrefix = null) - { - // The S3 log delivery group has to be added to the target bucket's ACP - if ($targetBucket !== null && ($acp = self::getAccessControlPolicy($targetBucket, '')) !== false) - { - // Only add permissions to the target bucket when they do not exist - $aclWriteSet = false; - $aclReadSet = false; - foreach ($acp['acl'] as $acl) - if ($acl['type'] == 'Group' && $acl['uri'] == 'http://acs.amazonaws.com/groups/s3/LogDelivery') - { - if ($acl['permission'] == 'WRITE') $aclWriteSet = true; - elseif ($acl['permission'] == 'READ_ACP') $aclReadSet = true; - } - if (!$aclWriteSet) $acp['acl'][] = array( - 'type' => 'Group', 'uri' => 'http://acs.amazonaws.com/groups/s3/LogDelivery', 'permission' => 'WRITE' - ); - if (!$aclReadSet) $acp['acl'][] = array( - 'type' => 'Group', 'uri' => 'http://acs.amazonaws.com/groups/s3/LogDelivery', 'permission' => 'READ_ACP' - ); - if (!$aclReadSet || !$aclWriteSet) self::setAccessControlPolicy($targetBucket, '', $acp); - } - - $dom = new DOMDocument; - $bucketLoggingStatus = $dom->createElement('BucketLoggingStatus'); - $bucketLoggingStatus->setAttribute('xmlns', 'http://s3.amazonaws.com/doc/2006-03-01/'); - if ($targetBucket !== null) - { - if ($targetPrefix == null) $targetPrefix = $bucket . '-'; - $loggingEnabled = $dom->createElement('LoggingEnabled'); - $loggingEnabled->appendChild($dom->createElement('TargetBucket', $targetBucket)); - $loggingEnabled->appendChild($dom->createElement('TargetPrefix', $targetPrefix)); - // TODO: Add TargetGrants? - $bucketLoggingStatus->appendChild($loggingEnabled); - } - $dom->appendChild($bucketLoggingStatus); - - $rest = new S3Request('PUT', $bucket, '', self::$endpoint); - $rest->setParameter('logging', null); - $rest->data = $dom->saveXML(); - $rest->size = strlen($rest->data); - $rest->setHeader('Content-Type', 'application/xml'); - $rest = $rest->getResponse(); - if ($rest->error === false && $rest->code !== 200) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - self::__triggerError(sprintf("S3::setBucketLogging({$bucket}, {$targetBucket}): [%s] %s", - $rest->error['code'], $rest->error['message']), __FILE__, __LINE__); - return false; - } - return true; - } - - - /** - * Get logging status for a bucket - * - * This will return false if logging is not enabled. - * Note: To enable logging, you also need to grant write access to the log group - * - * @param string $bucket Bucket name - * @return array | false - */ - public static function getBucketLogging($bucket) - { - $rest = new S3Request('GET', $bucket, '', self::$endpoint); - $rest->setParameter('logging', null); - $rest = $rest->getResponse(); - if ($rest->error === false && $rest->code !== 200) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - self::__triggerError(sprintf("S3::getBucketLogging({$bucket}): [%s] %s", - $rest->error['code'], $rest->error['message']), __FILE__, __LINE__); - return false; - } - if (!isset($rest->body->LoggingEnabled)) return false; // No logging - return array( - 'targetBucket' => (string)$rest->body->LoggingEnabled->TargetBucket, - 'targetPrefix' => (string)$rest->body->LoggingEnabled->TargetPrefix, - ); - } - - - /** - * Disable bucket logging - * - * @param string $bucket Bucket name - * @return boolean - */ - public static function disableBucketLogging($bucket) - { - return self::setBucketLogging($bucket, null); - } - - - /** - * Get a bucket's location - * - * @param string $bucket Bucket name - * @return string | false - */ - public static function getBucketLocation($bucket) - { - $rest = new S3Request('GET', $bucket, '', self::$endpoint); - $rest->setParameter('location', null); - $rest = $rest->getResponse(); - if ($rest->error === false && $rest->code !== 200) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - self::__triggerError(sprintf("S3::getBucketLocation({$bucket}): [%s] %s", - $rest->error['code'], $rest->error['message']), __FILE__, __LINE__); - return false; - } - return (isset($rest->body[0]) && (string)$rest->body[0] !== '') ? (string)$rest->body[0] : 'US'; - } - - - /** - * Set object or bucket Access Control Policy - * - * @param string $bucket Bucket name - * @param string $uri Object URI - * @param array $acp Access Control Policy Data (same as the data returned from getAccessControlPolicy) - * @return boolean - */ - public static function setAccessControlPolicy($bucket, $uri = '', $acp = array()) - { - $dom = new DOMDocument; - $dom->formatOutput = true; - $accessControlPolicy = $dom->createElement('AccessControlPolicy'); - $accessControlList = $dom->createElement('AccessControlList'); - - // It seems the owner has to be passed along too - $owner = $dom->createElement('Owner'); - $owner->appendChild($dom->createElement('ID', $acp['owner']['id'])); - $owner->appendChild($dom->createElement('DisplayName', $acp['owner']['name'])); - $accessControlPolicy->appendChild($owner); - - foreach ($acp['acl'] as $g) - { - $grant = $dom->createElement('Grant'); - $grantee = $dom->createElement('Grantee'); - $grantee->setAttribute('xmlns:xsi', 'http://www.w3.org/2001/XMLSchema-instance'); - if (isset($g['id'])) - { // CanonicalUser (DisplayName is omitted) - $grantee->setAttribute('xsi:type', 'CanonicalUser'); - $grantee->appendChild($dom->createElement('ID', $g['id'])); - } - elseif (isset($g['email'])) - { // AmazonCustomerByEmail - $grantee->setAttribute('xsi:type', 'AmazonCustomerByEmail'); - $grantee->appendChild($dom->createElement('EmailAddress', $g['email'])); - } - elseif ($g['type'] == 'Group') - { // Group - $grantee->setAttribute('xsi:type', 'Group'); - $grantee->appendChild($dom->createElement('URI', $g['uri'])); - } - $grant->appendChild($grantee); - $grant->appendChild($dom->createElement('Permission', $g['permission'])); - $accessControlList->appendChild($grant); - } - - $accessControlPolicy->appendChild($accessControlList); - $dom->appendChild($accessControlPolicy); - - $rest = new S3Request('PUT', $bucket, $uri, self::$endpoint); - $rest->setParameter('acl', null); - $rest->data = $dom->saveXML(); - $rest->size = strlen($rest->data); - $rest->setHeader('Content-Type', 'application/xml'); - $rest = $rest->getResponse(); - if ($rest->error === false && $rest->code !== 200) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - self::__triggerError(sprintf("S3::setAccessControlPolicy({$bucket}, {$uri}): [%s] %s", - $rest->error['code'], $rest->error['message']), __FILE__, __LINE__); - return false; - } - return true; - } - - - /** - * Get object or bucket Access Control Policy - * - * @param string $bucket Bucket name - * @param string $uri Object URI - * @return mixed | false - */ - public static function getAccessControlPolicy($bucket, $uri = '') - { - $rest = new S3Request('GET', $bucket, $uri, self::$endpoint); - $rest->setParameter('acl', null); - $rest = $rest->getResponse(); - if ($rest->error === false && $rest->code !== 200) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - self::__triggerError(sprintf("S3::getAccessControlPolicy({$bucket}, {$uri}): [%s] %s", - $rest->error['code'], $rest->error['message']), __FILE__, __LINE__); - return false; - } - - $acp = array(); - if (isset($rest->body->Owner, $rest->body->Owner->ID, $rest->body->Owner->DisplayName)) - $acp['owner'] = array( - 'id' => (string)$rest->body->Owner->ID, 'name' => (string)$rest->body->Owner->DisplayName - ); - - if (isset($rest->body->AccessControlList)) - { - $acp['acl'] = array(); - foreach ($rest->body->AccessControlList->Grant as $grant) - { - foreach ($grant->Grantee as $grantee) - { - if (isset($grantee->ID, $grantee->DisplayName)) // CanonicalUser - $acp['acl'][] = array( - 'type' => 'CanonicalUser', - 'id' => (string)$grantee->ID, - 'name' => (string)$grantee->DisplayName, - 'permission' => (string)$grant->Permission - ); - elseif (isset($grantee->EmailAddress)) // AmazonCustomerByEmail - $acp['acl'][] = array( - 'type' => 'AmazonCustomerByEmail', - 'email' => (string)$grantee->EmailAddress, - 'permission' => (string)$grant->Permission - ); - elseif (isset($grantee->URI)) // Group - $acp['acl'][] = array( - 'type' => 'Group', - 'uri' => (string)$grantee->URI, - 'permission' => (string)$grant->Permission - ); - else continue; - } - } - } - return $acp; - } - - - /** - * Delete an object - * - * @param string $bucket Bucket name - * @param string $uri Object URI - * @return boolean - */ - public static function deleteObject($bucket, $uri) - { - $rest = new S3Request('DELETE', $bucket, $uri, self::$endpoint); - $rest = $rest->getResponse(); - if ($rest->error === false && $rest->code !== 204) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - self::__triggerError(sprintf("S3::deleteObject(): [%s] %s", - $rest->error['code'], $rest->error['message']), __FILE__, __LINE__); - return false; - } - return true; - } - - - /** - * Get a query string authenticated URL - * - * @param string $bucket Bucket name - * @param string $uri Object URI - * @param integer $lifetime Lifetime in seconds - * @param boolean $hostBucket Use the bucket name as the hostname - * @param boolean $https Use HTTPS ($hostBucket should be false for SSL verification) - * @return string - */ - public static function getAuthenticatedURL($bucket, $uri, $lifetime, $hostBucket = false, $https = false) - { - $expires = time() + $lifetime; - $uri = str_replace(array('%2F', '%2B'), array('/', '+'), rawurlencode($uri)); - return sprintf(($https ? 'https' : 'http').'://%s/%s?AWSAccessKeyId=%s&Expires=%u&Signature=%s', - // $hostBucket ? $bucket : $bucket.'.s3.amazonaws.com', $uri, self::$__accessKey, $expires, - $hostBucket ? $bucket : self::$endpoint.'/'.$bucket, $uri, self::$__accessKey, $expires, - urlencode(self::__getHash("GET\n\n\n{$expires}\n/{$bucket}/{$uri}"))); - } - - - /** - * Get a CloudFront signed policy URL - * - * @param array $policy Policy - * @return string - */ - public static function getSignedPolicyURL($policy) - { - $data = json_encode($policy); - $signature = ''; - if (!openssl_sign($data, $signature, self::$__signingKeyResource)) return false; - - $encoded = str_replace(array('+', '='), array('-', '_', '~'), base64_encode($data)); - $signature = str_replace(array('+', '='), array('-', '_', '~'), base64_encode($signature)); - - $url = $policy['Statement'][0]['Resource'] . '?'; - foreach (array('Policy' => $encoded, 'Signature' => $signature, 'Key-Pair-Id' => self::$__signingKeyPairId) as $k => $v) - $url .= $k.'='.str_replace('%2F', '/', rawurlencode($v)).'&'; - return substr($url, 0, -1); - } - - - /** - * Get a CloudFront canned policy URL - * - * @param string $url URL to sign - * @param integer $lifetime URL lifetime - * @return string - */ - public static function getSignedCannedURL($url, $lifetime) - { - return self::getSignedPolicyURL(array( - 'Statement' => array( - array('Resource' => $url, 'Condition' => array( - 'DateLessThan' => array('AWS:EpochTime' => time() + $lifetime) - )) - ) - )); - } - - - /** - * Get upload POST parameters for form uploads - * - * @param string $bucket Bucket name - * @param string $uriPrefix Object URI prefix - * @param constant $acl ACL constant - * @param integer $lifetime Lifetime in seconds - * @param integer $maxFileSize Maximum filesize in bytes (default 5MB) - * @param string $successRedirect Redirect URL or 200 / 201 status code - * @param array $amzHeaders Array of x-amz-meta-* headers - * @param array $headers Array of request headers or content type as a string - * @param boolean $flashVars Includes additional "Filename" variable posted by Flash - * @return object - */ - public static function getHttpUploadPostParams($bucket, $uriPrefix = '', $acl = self::ACL_PRIVATE, $lifetime = 3600, - $maxFileSize = 5242880, $successRedirect = "201", $amzHeaders = array(), $headers = array(), $flashVars = false) - { - // Create policy object - $policy = new stdClass; - $policy->expiration = gmdate('Y-m-d\TH:i:s\Z', (time() + $lifetime)); - $policy->conditions = array(); - $obj = new stdClass; $obj->bucket = $bucket; array_push($policy->conditions, $obj); - $obj = new stdClass; $obj->acl = $acl; array_push($policy->conditions, $obj); - - $obj = new stdClass; // 200 for non-redirect uploads - if (is_numeric($successRedirect) && in_array((int)$successRedirect, array(200, 201))) - $obj->success_action_status = (string)$successRedirect; - else // URL - $obj->success_action_redirect = $successRedirect; - array_push($policy->conditions, $obj); - - if ($acl !== self::ACL_PUBLIC_READ) - array_push($policy->conditions, array('eq', '$acl', $acl)); - - array_push($policy->conditions, array('starts-with', '$key', $uriPrefix)); - if ($flashVars) array_push($policy->conditions, array('starts-with', '$Filename', '')); - foreach (array_keys($headers) as $headerKey) - array_push($policy->conditions, array('starts-with', '$'.$headerKey, '')); - foreach ($amzHeaders as $headerKey => $headerVal) - { - $obj = new stdClass; - $obj->{$headerKey} = (string)$headerVal; - array_push($policy->conditions, $obj); - } - array_push($policy->conditions, array('content-length-range', 0, $maxFileSize)); - $policy = base64_encode(str_replace('\/', '/', json_encode($policy))); - - // Create parameters - $params = new stdClass; - $params->AWSAccessKeyId = self::$__accessKey; - $params->key = $uriPrefix.'${filename}'; - $params->acl = $acl; - $params->policy = $policy; unset($policy); - $params->signature = self::__getHash($params->policy); - if (is_numeric($successRedirect) && in_array((int)$successRedirect, array(200, 201))) - $params->success_action_status = (string)$successRedirect; - else - $params->success_action_redirect = $successRedirect; - foreach ($headers as $headerKey => $headerVal) $params->{$headerKey} = (string)$headerVal; - foreach ($amzHeaders as $headerKey => $headerVal) $params->{$headerKey} = (string)$headerVal; - return $params; - } - - - /** - * Create a CloudFront distribution - * - * @param string $bucket Bucket name - * @param boolean $enabled Enabled (true/false) - * @param array $cnames Array containing CNAME aliases - * @param string $comment Use the bucket name as the hostname - * @param string $defaultRootObject Default root object - * @param string $originAccessIdentity Origin access identity - * @param array $trustedSigners Array of trusted signers - * @return array | false - */ - public static function createDistribution($bucket, $enabled = true, $cnames = array(), $comment = null, $defaultRootObject = null, $originAccessIdentity = null, $trustedSigners = array()) - { - if (!extension_loaded('openssl')) - { - self::__triggerError(sprintf("S3::createDistribution({$bucket}, ".(int)$enabled.", [], '$comment'): %s", - "CloudFront functionality requires SSL"), __FILE__, __LINE__); - return false; - } - $useSSL = self::$useSSL; - - self::$useSSL = true; // CloudFront requires SSL - $rest = new S3Request('POST', '', '2010-11-01/distribution', 'cloudfront.amazonaws.com'); - $rest->data = self::__getCloudFrontDistributionConfigXML( - $bucket.'.s3.amazonaws.com', - $enabled, - (string)$comment, - (string)microtime(true), - $cnames, - $defaultRootObject, - $originAccessIdentity, - $trustedSigners - ); - - $rest->size = strlen($rest->data); - $rest->setHeader('Content-Type', 'application/xml'); - $rest = self::__getCloudFrontResponse($rest); - - self::$useSSL = $useSSL; - - if ($rest->error === false && $rest->code !== 201) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - self::__triggerError(sprintf("S3::createDistribution({$bucket}, ".(int)$enabled.", [], '$comment'): [%s] %s", - $rest->error['code'], $rest->error['message']), __FILE__, __LINE__); - return false; - } elseif ($rest->body instanceof SimpleXMLElement) - return self::__parseCloudFrontDistributionConfig($rest->body); - return false; - } - - - /** - * Get CloudFront distribution info - * - * @param string $distributionId Distribution ID from listDistributions() - * @return array | false - */ - public static function getDistribution($distributionId) - { - if (!extension_loaded('openssl')) - { - self::__triggerError(sprintf("S3::getDistribution($distributionId): %s", - "CloudFront functionality requires SSL"), __FILE__, __LINE__); - return false; - } - $useSSL = self::$useSSL; - - self::$useSSL = true; // CloudFront requires SSL - $rest = new S3Request('GET', '', '2010-11-01/distribution/'.$distributionId, 'cloudfront.amazonaws.com'); - $rest = self::__getCloudFrontResponse($rest); - - self::$useSSL = $useSSL; - - if ($rest->error === false && $rest->code !== 200) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - self::__triggerError(sprintf("S3::getDistribution($distributionId): [%s] %s", - $rest->error['code'], $rest->error['message']), __FILE__, __LINE__); - return false; - } - elseif ($rest->body instanceof SimpleXMLElement) - { - $dist = self::__parseCloudFrontDistributionConfig($rest->body); - $dist['hash'] = $rest->headers['hash']; - $dist['id'] = $distributionId; - return $dist; - } - return false; - } - - - /** - * Update a CloudFront distribution - * - * @param array $dist Distribution array info identical to output of getDistribution() - * @return array | false - */ - public static function updateDistribution($dist) - { - if (!extension_loaded('openssl')) - { - self::__triggerError(sprintf("S3::updateDistribution({$dist['id']}): %s", - "CloudFront functionality requires SSL"), __FILE__, __LINE__); - return false; - } - - $useSSL = self::$useSSL; - - self::$useSSL = true; // CloudFront requires SSL - $rest = new S3Request('PUT', '', '2010-11-01/distribution/'.$dist['id'].'/config', 'cloudfront.amazonaws.com'); - $rest->data = self::__getCloudFrontDistributionConfigXML( - $dist['origin'], - $dist['enabled'], - $dist['comment'], - $dist['callerReference'], - $dist['cnames'], - $dist['defaultRootObject'], - $dist['originAccessIdentity'], - $dist['trustedSigners'] - ); - - $rest->size = strlen($rest->data); - $rest->setHeader('If-Match', $dist['hash']); - $rest = self::__getCloudFrontResponse($rest); - - self::$useSSL = $useSSL; - - if ($rest->error === false && $rest->code !== 200) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - self::__triggerError(sprintf("S3::updateDistribution({$dist['id']}): [%s] %s", - $rest->error['code'], $rest->error['message']), __FILE__, __LINE__); - return false; - } else { - $dist = self::__parseCloudFrontDistributionConfig($rest->body); - $dist['hash'] = $rest->headers['hash']; - return $dist; - } - return false; - } - - - /** - * Delete a CloudFront distribution - * - * @param array $dist Distribution array info identical to output of getDistribution() - * @return boolean - */ - public static function deleteDistribution($dist) - { - if (!extension_loaded('openssl')) - { - self::__triggerError(sprintf("S3::deleteDistribution({$dist['id']}): %s", - "CloudFront functionality requires SSL"), __FILE__, __LINE__); - return false; - } - - $useSSL = self::$useSSL; - - self::$useSSL = true; // CloudFront requires SSL - $rest = new S3Request('DELETE', '', '2008-06-30/distribution/'.$dist['id'], 'cloudfront.amazonaws.com'); - $rest->setHeader('If-Match', $dist['hash']); - $rest = self::__getCloudFrontResponse($rest); - - self::$useSSL = $useSSL; - - if ($rest->error === false && $rest->code !== 204) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - self::__triggerError(sprintf("S3::deleteDistribution({$dist['id']}): [%s] %s", - $rest->error['code'], $rest->error['message']), __FILE__, __LINE__); - return false; - } - return true; - } - - - /** - * Get a list of CloudFront distributions - * - * @return array - */ - public static function listDistributions() - { - if (!extension_loaded('openssl')) - { - self::__triggerError(sprintf("S3::listDistributions(): [%s] %s", - "CloudFront functionality requires SSL"), __FILE__, __LINE__); - return false; - } - - $useSSL = self::$useSSL; - self::$useSSL = true; // CloudFront requires SSL - $rest = new S3Request('GET', '', '2010-11-01/distribution', 'cloudfront.amazonaws.com'); - $rest = self::__getCloudFrontResponse($rest); - self::$useSSL = $useSSL; - - if ($rest->error === false && $rest->code !== 200) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - self::__triggerError(sprintf("S3::listDistributions(): [%s] %s", - $rest->error['code'], $rest->error['message']), __FILE__, __LINE__); - return false; - } - elseif ($rest->body instanceof SimpleXMLElement && isset($rest->body->DistributionSummary)) - { - $list = array(); - if (isset($rest->body->Marker, $rest->body->MaxItems, $rest->body->IsTruncated)) - { - //$info['marker'] = (string)$rest->body->Marker; - //$info['maxItems'] = (int)$rest->body->MaxItems; - //$info['isTruncated'] = (string)$rest->body->IsTruncated == 'true' ? true : false; - } - foreach ($rest->body->DistributionSummary as $summary) - $list[(string)$summary->Id] = self::__parseCloudFrontDistributionConfig($summary); - - return $list; - } - return array(); - } - - /** - * List CloudFront Origin Access Identities - * - * @return array - */ - public static function listOriginAccessIdentities() - { - if (!extension_loaded('openssl')) - { - self::__triggerError(sprintf("S3::listOriginAccessIdentities(): [%s] %s", - "CloudFront functionality requires SSL"), __FILE__, __LINE__); - return false; - } - - self::$useSSL = true; // CloudFront requires SSL - $rest = new S3Request('GET', '', '2010-11-01/origin-access-identity/cloudfront', 'cloudfront.amazonaws.com'); - $rest = self::__getCloudFrontResponse($rest); - $useSSL = self::$useSSL; - - if ($rest->error === false && $rest->code !== 200) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - trigger_error(sprintf("S3::listOriginAccessIdentities(): [%s] %s", - $rest->error['code'], $rest->error['message']), E_USER_WARNING); - return false; - } - - if (isset($rest->body->CloudFrontOriginAccessIdentitySummary)) - { - $identities = array(); - foreach ($rest->body->CloudFrontOriginAccessIdentitySummary as $identity) - if (isset($identity->S3CanonicalUserId)) - $identities[(string)$identity->Id] = array('id' => (string)$identity->Id, 's3CanonicalUserId' => (string)$identity->S3CanonicalUserId); - return $identities; - } - return false; - } - - - /** - * Invalidate objects in a CloudFront distribution - * - * Thanks to Martin Lindkvist for S3::invalidateDistribution() - * - * @param string $distributionId Distribution ID from listDistributions() - * @param array $paths Array of object paths to invalidate - * @return boolean - */ - public static function invalidateDistribution($distributionId, $paths) - { - if (!extension_loaded('openssl')) - { - self::__triggerError(sprintf("S3::invalidateDistribution(): [%s] %s", - "CloudFront functionality requires SSL"), __FILE__, __LINE__); - return false; - } - - $useSSL = self::$useSSL; - self::$useSSL = true; // CloudFront requires SSL - $rest = new S3Request('POST', '', '2010-08-01/distribution/'.$distributionId.'/invalidation', 'cloudfront.amazonaws.com'); - $rest->data = self::__getCloudFrontInvalidationBatchXML($paths, (string)microtime(true)); - $rest->size = strlen($rest->data); - $rest = self::__getCloudFrontResponse($rest); - self::$useSSL = $useSSL; - - if ($rest->error === false && $rest->code !== 201) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - trigger_error(sprintf("S3::invalidate('{$distributionId}',{$paths}): [%s] %s", - $rest->error['code'], $rest->error['message']), E_USER_WARNING); - return false; - } - return true; - } - - - /** - * Get a InvalidationBatch DOMDocument - * - * @internal Used to create XML in invalidateDistribution() - * @param array $paths Paths to objects to invalidateDistribution - * @param int $callerReference - * @return string - */ - private static function __getCloudFrontInvalidationBatchXML($paths, $callerReference = '0') - { - $dom = new DOMDocument('1.0', 'UTF-8'); - $dom->formatOutput = true; - $invalidationBatch = $dom->createElement('InvalidationBatch'); - foreach ($paths as $path) - $invalidationBatch->appendChild($dom->createElement('Path', $path)); - - $invalidationBatch->appendChild($dom->createElement('CallerReference', $callerReference)); - $dom->appendChild($invalidationBatch); - return $dom->saveXML(); - } - - - /** - * List your invalidation batches for invalidateDistribution() in a CloudFront distribution - * - * http://docs.amazonwebservices.com/AmazonCloudFront/latest/APIReference/ListInvalidation.html - * returned array looks like this: - * Array - * ( - * [I31TWB0CN9V6XD] => InProgress - * [IT3TFE31M0IHZ] => Completed - * [I12HK7MPO1UQDA] => Completed - * [I1IA7R6JKTC3L2] => Completed - * ) - * - * @param string $distributionId Distribution ID from listDistributions() - * @return array - */ - public static function getDistributionInvalidationList($distributionId) - { - if (!extension_loaded('openssl')) - { - self::__triggerError(sprintf("S3::getDistributionInvalidationList(): [%s] %s", - "CloudFront functionality requires SSL"), __FILE__, __LINE__); - return false; - } - - $useSSL = self::$useSSL; - self::$useSSL = true; // CloudFront requires SSL - $rest = new S3Request('GET', '', '2010-11-01/distribution/'.$distributionId.'/invalidation', 'cloudfront.amazonaws.com'); - $rest = self::__getCloudFrontResponse($rest); - self::$useSSL = $useSSL; - - if ($rest->error === false && $rest->code !== 200) - $rest->error = array('code' => $rest->code, 'message' => 'Unexpected HTTP status'); - if ($rest->error !== false) - { - trigger_error(sprintf("S3::getDistributionInvalidationList('{$distributionId}'): [%s]", - $rest->error['code'], $rest->error['message']), E_USER_WARNING); - return false; - } - elseif ($rest->body instanceof SimpleXMLElement && isset($rest->body->InvalidationSummary)) - { - $list = array(); - foreach ($rest->body->InvalidationSummary as $summary) - $list[(string)$summary->Id] = (string)$summary->Status; - - return $list; - } - return array(); - } - - - /** - * Get a DistributionConfig DOMDocument - * - * http://docs.amazonwebservices.com/AmazonCloudFront/latest/APIReference/index.html?PutConfig.html - * - * @internal Used to create XML in createDistribution() and updateDistribution() - * @param string $bucket S3 Origin bucket - * @param boolean $enabled Enabled (true/false) - * @param string $comment Comment to append - * @param string $callerReference Caller reference - * @param array $cnames Array of CNAME aliases - * @param string $defaultRootObject Default root object - * @param string $originAccessIdentity Origin access identity - * @param array $trustedSigners Array of trusted signers - * @return string - */ - private static function __getCloudFrontDistributionConfigXML($bucket, $enabled, $comment, $callerReference = '0', $cnames = array(), $defaultRootObject = null, $originAccessIdentity = null, $trustedSigners = array()) - { - $dom = new DOMDocument('1.0', 'UTF-8'); - $dom->formatOutput = true; - $distributionConfig = $dom->createElement('DistributionConfig'); - $distributionConfig->setAttribute('xmlns', 'http://cloudfront.amazonaws.com/doc/2010-11-01/'); - - $origin = $dom->createElement('S3Origin'); - $origin->appendChild($dom->createElement('DNSName', $bucket)); - if ($originAccessIdentity !== null) $origin->appendChild($dom->createElement('OriginAccessIdentity', $originAccessIdentity)); - $distributionConfig->appendChild($origin); - - if ($defaultRootObject !== null) $distributionConfig->appendChild($dom->createElement('DefaultRootObject', $defaultRootObject)); - - $distributionConfig->appendChild($dom->createElement('CallerReference', $callerReference)); - foreach ($cnames as $cname) - $distributionConfig->appendChild($dom->createElement('CNAME', $cname)); - if ($comment !== '') $distributionConfig->appendChild($dom->createElement('Comment', $comment)); - $distributionConfig->appendChild($dom->createElement('Enabled', $enabled ? 'true' : 'false')); - - $trusted = $dom->createElement('TrustedSigners'); - foreach ($trustedSigners as $id => $type) - $trusted->appendChild($id !== '' ? $dom->createElement($type, $id) : $dom->createElement($type)); - $distributionConfig->appendChild($trusted); - - $dom->appendChild($distributionConfig); - //var_dump($dom->saveXML()); - return $dom->saveXML(); - } - - - /** - * Parse a CloudFront distribution config - * - * See http://docs.amazonwebservices.com/AmazonCloudFront/latest/APIReference/index.html?GetDistribution.html - * - * @internal Used to parse the CloudFront DistributionConfig node to an array - * @param object &$node DOMNode - * @return array - */ - private static function __parseCloudFrontDistributionConfig(&$node) - { - if (isset($node->DistributionConfig)) - return self::__parseCloudFrontDistributionConfig($node->DistributionConfig); - - $dist = array(); - if (isset($node->Id, $node->Status, $node->LastModifiedTime, $node->DomainName)) - { - $dist['id'] = (string)$node->Id; - $dist['status'] = (string)$node->Status; - $dist['time'] = strtotime((string)$node->LastModifiedTime); - $dist['domain'] = (string)$node->DomainName; - } - - if (isset($node->CallerReference)) - $dist['callerReference'] = (string)$node->CallerReference; - - if (isset($node->Enabled)) - $dist['enabled'] = (string)$node->Enabled == 'true' ? true : false; - - if (isset($node->S3Origin)) - { - if (isset($node->S3Origin->DNSName)) - $dist['origin'] = (string)$node->S3Origin->DNSName; - - $dist['originAccessIdentity'] = isset($node->S3Origin->OriginAccessIdentity) ? - (string)$node->S3Origin->OriginAccessIdentity : null; - } - - $dist['defaultRootObject'] = isset($node->DefaultRootObject) ? (string)$node->DefaultRootObject : null; - - $dist['cnames'] = array(); - if (isset($node->CNAME)) - foreach ($node->CNAME as $cname) - $dist['cnames'][(string)$cname] = (string)$cname; - - $dist['trustedSigners'] = array(); - if (isset($node->TrustedSigners)) - foreach ($node->TrustedSigners as $signer) - { - if (isset($signer->Self)) - $dist['trustedSigners'][''] = 'Self'; - elseif (isset($signer->KeyPairId)) - $dist['trustedSigners'][(string)$signer->KeyPairId] = 'KeyPairId'; - elseif (isset($signer->AwsAccountNumber)) - $dist['trustedSigners'][(string)$signer->AwsAccountNumber] = 'AwsAccountNumber'; - } - - $dist['comment'] = isset($node->Comment) ? (string)$node->Comment : null; - return $dist; - } - - - /** - * Grab CloudFront response - * - * @internal Used to parse the CloudFront S3Request::getResponse() output - * @param object &$rest S3Request instance - * @return object - */ - private static function __getCloudFrontResponse(&$rest) - { - $rest->getResponse(); - if ($rest->response->error === false && isset($rest->response->body) && - is_string($rest->response->body) && substr($rest->response->body, 0, 5) == 'response->body = simplexml_load_string($rest->response->body); - // Grab CloudFront errors - if (isset($rest->response->body->Error, $rest->response->body->Error->Code, - $rest->response->body->Error->Message)) - { - $rest->response->error = array( - 'code' => (string)$rest->response->body->Error->Code, - 'message' => (string)$rest->response->body->Error->Message - ); - unset($rest->response->body); - } - } - return $rest->response; - } - - - /** - * Get MIME type for file - * - * To override the putObject() Content-Type, add it to $requestHeaders - * - * To use fileinfo, ensure the MAGIC environment variable is set - * - * @internal Used to get mime types - * @param string &$file File path - * @return string - */ - private static function __getMimeType(&$file) - { - // Use fileinfo if available - if (extension_loaded('fileinfo') && isset($_ENV['MAGIC']) && - ($finfo = finfo_open(FILEINFO_MIME, $_ENV['MAGIC'])) !== false) - { - if (($type = finfo_file($finfo, $file)) !== false) - { - // Remove the charset and grab the last content-type - $type = explode(' ', str_replace('; charset=', ';charset=', $type)); - $type = array_pop($type); - $type = explode(';', $type); - $type = trim(array_shift($type)); - } - finfo_close($finfo); - if ($type !== false && strlen($type) > 0) return $type; - } - - static $exts = array( - 'jpg' => 'image/jpeg', 'jpeg' => 'image/jpeg', 'gif' => 'image/gif', - 'png' => 'image/png', 'ico' => 'image/x-icon', 'pdf' => 'application/pdf', - 'tif' => 'image/tiff', 'tiff' => 'image/tiff', 'svg' => 'image/svg+xml', - 'svgz' => 'image/svg+xml', 'swf' => 'application/x-shockwave-flash', - 'zip' => 'application/zip', 'gz' => 'application/x-gzip', - 'tar' => 'application/x-tar', 'bz' => 'application/x-bzip', - 'bz2' => 'application/x-bzip2', 'rar' => 'application/x-rar-compressed', - 'exe' => 'application/x-msdownload', 'msi' => 'application/x-msdownload', - 'cab' => 'application/vnd.ms-cab-compressed', 'txt' => 'text/plain', - 'asc' => 'text/plain', 'htm' => 'text/html', 'html' => 'text/html', - 'css' => 'text/css', 'js' => 'text/javascript', - 'xml' => 'text/xml', 'xsl' => 'application/xsl+xml', - 'ogg' => 'application/ogg', 'mp3' => 'audio/mpeg', 'wav' => 'audio/x-wav', - 'avi' => 'video/x-msvideo', 'mpg' => 'video/mpeg', 'mpeg' => 'video/mpeg', - 'mov' => 'video/quicktime', 'flv' => 'video/x-flv', 'php' => 'text/x-php' - ); - $ext = strtolower(pathinfo($file, PATHINFO_EXTENSION)); - // mime_content_type() is deprecated, fileinfo should be configured - $type = isset($exts[$ext]) ? $exts[$ext] : trim(mime_content_type($file)); - - return ($type !== false && strlen($type) > 0) ? $type : 'application/octet-stream'; - } - - - /** - * Generate the auth string: "AWS AccessKey:Signature" - * - * @internal Used by S3Request::getResponse() - * @param string $string String to sign - * @return string - */ - public static function __getSignature($string) - { - return 'AWS '.self::$__accessKey.':'.self::__getHash($string); - } - - - /** - * Creates a HMAC-SHA1 hash - * - * This uses the hash extension if loaded - * - * @internal Used by __getSignature() - * @param string $string String to sign - * @return string - */ - private static function __getHash($string) - { - return base64_encode(extension_loaded('hash') ? - hash_hmac('sha1', $string, self::$__secretKey, true) : pack('H*', sha1( - (str_pad(self::$__secretKey, 64, chr(0x00)) ^ (str_repeat(chr(0x5c), 64))) . - pack('H*', sha1((str_pad(self::$__secretKey, 64, chr(0x00)) ^ - (str_repeat(chr(0x36), 64))) . $string))))); - } - -} - -/** - * S3 Request class - * - * @link http://undesigned.org.za/2007/10/22/amazon-s3-php-class - * @version 0.5.0-dev - */ -final class S3Request -{ - /** - * AWS URI - * - * @var string - * @access pricate - */ - private $endpoint; - - /** - * Verb - * - * @var string - * @access private - */ - private $verb; - - /** - * S3 bucket name - * - * @var string - * @access private - */ - private $bucket; - - /** - * Object URI - * - * @var string - * @access private - */ - private $uri; - - /** - * Final object URI - * - * @var string - * @access private - */ - private $resource = ''; - - /** - * Additional request parameters - * - * @var array - * @access private - */ - private $parameters = array(); - - /** - * Amazon specific request headers - * - * @var array - * @access private - */ - private $amzHeaders = array(); - - /** - * HTTP request headers - * - * @var array - * @access private - */ - private $headers = array( - 'Host' => '', 'Date' => '', 'Content-MD5' => '', 'Content-Type' => '' - ); - - /** - * Use HTTP PUT? - * - * @var bool - * @access public - */ - public $fp = false; - - /** - * PUT file size - * - * @var int - * @access public - */ - public $size = 0; - - /** - * PUT post fields - * - * @var array - * @access public - */ - public $data = false; - - /** - * S3 request respone - * - * @var object - * @access public - */ - public $response; - - - /** - * Constructor - * - * @param string $verb Verb - * @param string $bucket Bucket name - * @param string $uri Object URI - * @param string $endpoint AWS endpoint URI - * @return mixed - */ - function __construct($verb, $bucket = '', $uri = '', $endpoint = 's3.amazonaws.com') - { - - $this->endpoint = $endpoint; - $this->verb = $verb; - $this->bucket = $bucket; - $this->uri = $uri !== '' ? '/'.str_replace('%2F', '/', rawurlencode($uri)) : '/'; - - //if ($this->bucket !== '') - // $this->resource = '/'.$this->bucket.$this->uri; - //else - // $this->resource = $this->uri; - - if ($this->bucket !== '') - { - if ($this->__dnsBucketName($this->bucket)) - { - $this->headers['Host'] = $this->bucket.'.'.$this->endpoint; - $this->resource = '/'.$this->bucket.$this->uri; - } - else - { - $this->headers['Host'] = $this->endpoint; - $this->uri = $this->uri; - if ($this->bucket !== '') $this->uri = '/'.$this->bucket.$this->uri; - $this->bucket = ''; - $this->resource = $this->uri; - } - } - else - { - $this->headers['Host'] = $this->endpoint; - $this->resource = $this->uri; - } - - - $this->headers['Date'] = gmdate('D, d M Y H:i:s T'); - $this->response = new STDClass; - $this->response->error = false; - $this->response->body = null; - $this->response->headers = array(); - } - - - /** - * Set request parameter - * - * @param string $key Key - * @param string $value Value - * @return void - */ - public function setParameter($key, $value) - { - $this->parameters[$key] = $value; - } - - - /** - * Set request header - * - * @param string $key Key - * @param string $value Value - * @return void - */ - public function setHeader($key, $value) - { - $this->headers[$key] = $value; - } - - - /** - * Set x-amz-meta-* header - * - * @param string $key Key - * @param string $value Value - * @return void - */ - public function setAmzHeader($key, $value) - { - $this->amzHeaders[$key] = $value; - } - - - /** - * Get the S3 response - * - * @return object | false - */ - public function getResponse() - { - $query = ''; - if (sizeof($this->parameters) > 0) - { - $query = substr($this->uri, -1) !== '?' ? '?' : '&'; - foreach ($this->parameters as $var => $value) - if ($value == null || $value == '') $query .= $var.'&'; - else $query .= $var.'='.rawurlencode($value).'&'; - $query = substr($query, 0, -1); - $this->uri .= $query; - - if (array_key_exists('acl', $this->parameters) || - array_key_exists('location', $this->parameters) || - array_key_exists('torrent', $this->parameters) || - array_key_exists('website', $this->parameters) || - array_key_exists('logging', $this->parameters)) - $this->resource .= $query; - } - $url = (S3::$useSSL ? 'https://' : 'http://') . ($this->headers['Host'] !== '' ? $this->headers['Host'] : $this->endpoint) . $this->uri; - - //var_dump('bucket: ' . $this->bucket, 'uri: ' . $this->uri, 'resource: ' . $this->resource, 'url: ' . $url); - - // Basic setup - $curl = curl_init(); - curl_setopt($curl, CURLOPT_USERAGENT, 'S3/php'); - - if (S3::$useSSL) - { - // SSL Validation can now be optional for those with broken OpenSSL installations - curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, S3::$useSSLValidation ? 2 : 0); - curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, S3::$useSSLValidation ? 1 : 0); - - if (S3::$sslKey !== null) curl_setopt($curl, CURLOPT_SSLKEY, S3::$sslKey); - if (S3::$sslCert !== null) curl_setopt($curl, CURLOPT_SSLCERT, S3::$sslCert); - if (S3::$sslCACert !== null) curl_setopt($curl, CURLOPT_CAINFO, S3::$sslCACert); - } - - curl_setopt($curl, CURLOPT_URL, $url); - - if (S3::$proxy != null && isset(S3::$proxy['host'])) - { - curl_setopt($curl, CURLOPT_PROXY, S3::$proxy['host']); - curl_setopt($curl, CURLOPT_PROXYTYPE, S3::$proxy['type']); - if (isset(S3::$proxy['user'], S3::$proxy['pass']) && S3::$proxy['user'] != null && S3::$proxy['pass'] != null) - curl_setopt($curl, CURLOPT_PROXYUSERPWD, sprintf('%s:%s', S3::$proxy['user'], S3::$proxy['pass'])); - } - - // Headers - $headers = array(); $amz = array(); - foreach ($this->amzHeaders as $header => $value) - if (strlen($value) > 0) $headers[] = $header.': '.$value; - foreach ($this->headers as $header => $value) - if (strlen($value) > 0) $headers[] = $header.': '.$value; - - // Collect AMZ headers for signature - foreach ($this->amzHeaders as $header => $value) - if (strlen($value) > 0) $amz[] = strtolower($header).':'.$value; - - // AMZ headers must be sorted - if (sizeof($amz) > 0) - { - //sort($amz); - usort($amz, array(&$this, '__sortMetaHeadersCmp')); - $amz = "\n".implode("\n", $amz); - } else $amz = ''; - - if (S3::hasAuth()) - { - // Authorization string (CloudFront stringToSign should only contain a date) - if ($this->headers['Host'] == 'cloudfront.amazonaws.com') - $headers[] = 'Authorization: ' . S3::__getSignature($this->headers['Date']); - else - { - $headers[] = 'Authorization: ' . S3::__getSignature( - $this->verb."\n". - $this->headers['Content-MD5']."\n". - $this->headers['Content-Type']."\n". - $this->headers['Date'].$amz."\n". - $this->resource - ); - } - } - - curl_setopt($curl, CURLOPT_HTTPHEADER, $headers); - curl_setopt($curl, CURLOPT_HEADER, false); - curl_setopt($curl, CURLOPT_RETURNTRANSFER, false); - curl_setopt($curl, CURLOPT_WRITEFUNCTION, array(&$this, '__responseWriteCallback')); - curl_setopt($curl, CURLOPT_HEADERFUNCTION, array(&$this, '__responseHeaderCallback')); - curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); - - // Request types - switch ($this->verb) - { - case 'GET': break; - case 'PUT': case 'POST': // POST only used for CloudFront - if ($this->fp !== false) - { - curl_setopt($curl, CURLOPT_PUT, true); - curl_setopt($curl, CURLOPT_INFILE, $this->fp); - if ($this->size >= 0) - curl_setopt($curl, CURLOPT_INFILESIZE, $this->size); - } - elseif ($this->data !== false) - { - curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $this->verb); - curl_setopt($curl, CURLOPT_POSTFIELDS, $this->data); - } - else - curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $this->verb); - break; - case 'HEAD': - curl_setopt($curl, CURLOPT_CUSTOMREQUEST, 'HEAD'); - curl_setopt($curl, CURLOPT_NOBODY, true); - break; - case 'DELETE': - curl_setopt($curl, CURLOPT_CUSTOMREQUEST, 'DELETE'); - break; - default: break; - } - - // Execute, grab errors - if (curl_exec($curl)) - $this->response->code = curl_getinfo($curl, CURLINFO_HTTP_CODE); - else - $this->response->error = array( - 'code' => curl_errno($curl), - 'message' => curl_error($curl), - 'resource' => $this->resource - ); - - @curl_close($curl); - - // Parse body into XML - if ($this->response->error === false && isset($this->response->headers['type']) && - $this->response->headers['type'] == 'application/xml' && isset($this->response->body)) - { - $this->response->body = simplexml_load_string($this->response->body); - - // Grab S3 errors - if (!in_array($this->response->code, array(200, 204, 206)) && - isset($this->response->body->Code, $this->response->body->Message)) - { - $this->response->error = array( - 'code' => (string)$this->response->body->Code, - 'message' => (string)$this->response->body->Message - ); - if (isset($this->response->body->Resource)) - $this->response->error['resource'] = (string)$this->response->body->Resource; - unset($this->response->body); - } - } - - // Clean up file resources - if ($this->fp !== false && is_resource($this->fp)) fclose($this->fp); - - return $this->response; - } - - /** - * Sort compare for meta headers - * - * @internal Used to sort x-amz meta headers - * @param string $a String A - * @param string $b String B - * @return integer - */ - private function __sortMetaHeadersCmp($a, $b) - { - $lenA = strpos($a, ':'); - $lenB = strpos($b, ':'); - $minLen = min($lenA, $lenB); - $ncmp = strncmp($a, $b, $minLen); - if ($lenA == $lenB) return $ncmp; - if (0 == $ncmp) return $lenA < $lenB ? -1 : 1; - return $ncmp; - } - - /** - * CURL write callback - * - * @param resource &$curl CURL resource - * @param string &$data Data - * @return integer - */ - private function __responseWriteCallback(&$curl, &$data) - { - if (in_array($this->response->code, array(200, 206)) && $this->fp !== false) - return fwrite($this->fp, $data); - else - $this->response->body .= $data; - return strlen($data); - } - - - /** - * Check DNS conformity - * - * @param string $bucket Bucket name - * @return boolean - */ - private function __dnsBucketName($bucket) - { - if (strlen($bucket) > 63 || preg_match("/[^a-z0-9\.-]/", $bucket) > 0) return false; - if (strstr($bucket, '-.') !== false) return false; - if (strstr($bucket, '..') !== false) return false; - if (!preg_match("/^[0-9a-z]/", $bucket)) return false; - if (!preg_match("/[0-9a-z]$/", $bucket)) return false; - return true; - } - - - /** - * CURL header callback - * - * @param resource &$curl CURL resource - * @param string &$data Data - * @return integer - */ - private function __responseHeaderCallback(&$curl, &$data) - { - if (($strlen = strlen($data)) <= 2) return $strlen; - if (substr($data, 0, 4) == 'HTTP') - $this->response->code = (int)substr($data, 9, 3); - else - { - $data = trim($data); - if (strpos($data, ': ') === false) return $strlen; - list($header, $value) = explode(': ', $data, 2); - if ($header == 'Last-Modified') - $this->response->headers['time'] = strtotime($value); - elseif ($header == 'Content-Length') - $this->response->headers['size'] = (int)$value; - elseif ($header == 'Content-Type') - $this->response->headers['type'] = $value; - elseif ($header == 'ETag') - $this->response->headers['hash'] = $value{0} == '"' ? substr($value, 1, -1) : $value; - elseif (preg_match('/^x-amz-meta-.*$/', $header)) - $this->response->headers[$header] = $value; - } - return $strlen; - } - -} - -/** - * S3 exception class - * - * @link http://undesigned.org.za/2007/10/22/amazon-s3-php-class - * @version 0.5.0-dev - */ - -class S3Exception extends Exception { - /** - * Class constructor - * - * @param string $message Exception message - * @param string $file File in which exception was created - * @param string $line Line number on which exception was created - * @param int $code Exception code - */ - function __construct($message, $file, $line, $code = 0) - { - parent::__construct($message, $code); - $this->file = $file; - $this->line = $line; - } -} diff --git a/src/applications/config/check/PhabricatorStorageSetupCheck.php b/src/applications/config/check/PhabricatorStorageSetupCheck.php index dbe79c90d6..9ec753da05 100644 --- a/src/applications/config/check/PhabricatorStorageSetupCheck.php +++ b/src/applications/config/check/PhabricatorStorageSetupCheck.php @@ -13,6 +13,8 @@ final class PhabricatorStorageSetupCheck extends PhabricatorSetupCheck { $engines = PhabricatorFileStorageEngine::loadWritableChunkEngines(); $chunk_engine_active = (bool)$engines; + $this->checkS3(); + if (!$chunk_engine_active) { $doc_href = PhabricatorEnv::getDocLink('Configuring File Storage'); @@ -140,4 +142,55 @@ final class PhabricatorStorageSetupCheck extends PhabricatorSetupCheck { ->addPhabricatorConfig('storage.local-disk.path'); } } + + private function checkS3() { + $access_key = PhabricatorEnv::getEnvConfig('amazon-s3.access-key'); + $secret_key = PhabricatorEnv::getEnvConfig('amazon-s3.secret-key'); + $region = PhabricatorEnv::getEnvConfig('amazon-s3.region'); + $endpoint = PhabricatorEnv::getEnvConfig('amazon-s3.endpoint'); + + $how_many = 0; + + if (strlen($access_key)) { + $how_many++; + } + + if (strlen($secret_key)) { + $how_many++; + } + + if (strlen($region)) { + $how_many++; + } + + if (strlen($endpoint)) { + $how_many++; + } + + // Nothing configured, no issues here. + if ($how_many === 0) { + return; + } + + // Everything configured, no issues here. + if ($how_many === 4) { + return; + } + + $message = pht( + 'File storage in Amazon S3 has been partially configured, but you are '. + 'missing some required settings. S3 will not be available to store '. + 'files until you complete the configuration. Either configure S3 fully '. + 'or remove the partial configuration.'); + + $this->newIssue('storage.s3.partial-config') + ->setShortName(pht('S3 Partially Configured')) + ->setName(pht('Amazon S3 is Only Partially Configured')) + ->setMessage($message) + ->addPhabricatorConfig('amazon-s3.access-key') + ->addPhabricatorConfig('amazon-s3.secret-key') + ->addPhabricatorConfig('amazon-s3.region') + ->addPhabricatorConfig('amazon-s3.endpoint'); + } + } diff --git a/src/applications/config/option/PhabricatorAWSConfigOptions.php b/src/applications/config/option/PhabricatorAWSConfigOptions.php index b191e0e40e..af328ba0af 100644 --- a/src/applications/config/option/PhabricatorAWSConfigOptions.php +++ b/src/applications/config/option/PhabricatorAWSConfigOptions.php @@ -33,14 +33,27 @@ final class PhabricatorAWSConfigOptions $this->newOption('amazon-s3.secret-key', 'string', null) ->setHidden(true) ->setDescription(pht('Secret key for Amazon S3.')), + $this->newOption('amazon-s3.region', 'string', null) + ->setLocked(true) + ->setDescription( + pht( + 'Amazon S3 region where your S3 bucket is located. When you '. + 'specify a region, you should also specify a corresponding '. + 'endpoint with `amazon-s3.endpoint`. You can find a list of '. + 'available regions and endpoints in the AWS documentation.')) + ->addExample('us-west-1', pht('USWest Region')), $this->newOption('amazon-s3.endpoint', 'string', null) ->setLocked(true) ->setDescription( pht( - 'Explicit S3 endpoint to use. Leave empty to have Phabricator '. - 'select and endpoint. Normally, you do not need to set this.')) - ->addExample(null, pht('Use default endpoint')) - ->addExample('s3.amazon.com', pht('Use specific endpoint')), + 'Explicit S3 endpoint to use. This should be the endpoint '. + 'which corresponds to the region you have selected in '. + '`amazon-s3.region`. Phabricator can not determine the correct '. + 'endpoint automatically because some endpoint locations are '. + 'irregular.')) + ->addExample( + 's3-us-west-1.amazonaws.com', + pht('Use specific endpoint')), $this->newOption('amazon-ec2.access-key', 'string', null) ->setLocked(true) ->setDescription(pht('Access key for Amazon EC2.')), diff --git a/src/applications/config/view/PhabricatorSetupIssueView.php b/src/applications/config/view/PhabricatorSetupIssueView.php index ed82ffa7b7..a2a703c436 100644 --- a/src/applications/config/view/PhabricatorSetupIssueView.php +++ b/src/applications/config/view/PhabricatorSetupIssueView.php @@ -275,20 +275,21 @@ final class PhabricatorSetupIssueView extends AphrontView { $update = array(); foreach ($configs as $config) { if (idx($options, $config) && $options[$config]->getLocked()) { - continue; + $name = pht('View "%s"', $config); + } else { + $name = pht('Edit "%s"', $config); } $link = phutil_tag( 'a', array( 'href' => '/config/edit/'.$config.'/?issue='.$issue->getIssueKey(), ), - pht('Edit %s', $config)); + $name); $update[] = phutil_tag('li', array(), $link); } if ($update) { $update = phutil_tag('ul', array(), $update); if (!$related) { - $update_info = phutil_tag( 'p', array(), diff --git a/src/applications/files/engine/PhabricatorS3FileStorageEngine.php b/src/applications/files/engine/PhabricatorS3FileStorageEngine.php index 59446c2c13..443a5a9ddc 100644 --- a/src/applications/files/engine/PhabricatorS3FileStorageEngine.php +++ b/src/applications/files/engine/PhabricatorS3FileStorageEngine.php @@ -28,8 +28,14 @@ final class PhabricatorS3FileStorageEngine $bucket = PhabricatorEnv::getEnvConfig('storage.s3.bucket'); $access_key = PhabricatorEnv::getEnvConfig('amazon-s3.access-key'); $secret_key = PhabricatorEnv::getEnvConfig('amazon-s3.secret-key'); + $endpoint = PhabricatorEnv::getEnvConfig('amazon-s3.endpoint'); + $region = PhabricatorEnv::getEnvConfig('amazon-s3.region'); - return (strlen($bucket) && strlen($access_key) && strlen($secret_key)); + return (strlen($bucket) && + strlen($access_key) && + strlen($secret_key) && + strlen($endpoint) && + strlen($region)); } @@ -68,11 +74,11 @@ final class PhabricatorS3FileStorageEngine 'type' => 's3', 'method' => 'putObject', )); - $s3->putObject( - $data, - $this->getBucketName(), - $name, - $acl = 'private'); + + $s3 + ->setParametersForPutObject($name, $data) + ->resolve(); + $profiler->endServiceCall($call_id, array()); return $name; @@ -84,24 +90,21 @@ final class PhabricatorS3FileStorageEngine */ public function readFile($handle) { $s3 = $this->newS3API(); + $profiler = PhutilServiceProfiler::getInstance(); $call_id = $profiler->beginServiceCall( array( 'type' => 's3', 'method' => 'getObject', )); - $result = $s3->getObject( - $this->getBucketName(), - $handle); + + $result = $s3 + ->setParametersForGetObject($handle) + ->resolve(); + $profiler->endServiceCall($call_id, array()); - // NOTE: The implementation of the API that we're using may respond with - // a successful result that has length 0 and no body property. - if (isset($result->body)) { - return $result->body; - } else { - return ''; - } + return $result; } @@ -109,17 +112,20 @@ final class PhabricatorS3FileStorageEngine * Delete a blob from Amazon S3. */ public function deleteFile($handle) { - AphrontWriteGuard::willWrite(); $s3 = $this->newS3API(); + + AphrontWriteGuard::willWrite(); $profiler = PhutilServiceProfiler::getInstance(); $call_id = $profiler->beginServiceCall( array( 'type' => 's3', 'method' => 'deleteObject', )); - $s3->deleteObject( - $this->getBucketName(), - $handle); + + $s3 + ->setParametersForDeleteObject($handle) + ->resolve(); + $profiler->endServiceCall($call_id, array()); } @@ -147,33 +153,19 @@ final class PhabricatorS3FileStorageEngine * Create a new S3 API object. * * @task internal - * @phutil-external-symbol class S3 */ private function newS3API() { - $libroot = dirname(phutil_get_library_root('phabricator')); - require_once $libroot.'/externals/s3/S3.php'; - $access_key = PhabricatorEnv::getEnvConfig('amazon-s3.access-key'); $secret_key = PhabricatorEnv::getEnvConfig('amazon-s3.secret-key'); + $region = PhabricatorEnv::getEnvConfig('amazon-s3.region'); $endpoint = PhabricatorEnv::getEnvConfig('amazon-s3.endpoint'); - if (!$access_key || !$secret_key) { - throw new PhabricatorFileStorageConfigurationException( - pht( - "Specify '%s' and '%s'!", - 'amazon-s3.access-key', - 'amazon-s3.secret-key')); - } - - if ($endpoint !== null) { - $s3 = new S3($access_key, $secret_key, $use_ssl = true, $endpoint); - } else { - $s3 = new S3($access_key, $secret_key, $use_ssl = true); - } - - $s3->setExceptions(true); - - return $s3; + return id(new PhutilAWSS3Future()) + ->setAccessKey($access_key) + ->setSecretKey(new PhutilOpaqueEnvelope($secret_key)) + ->setRegion($region) + ->setEndpoint($endpoint) + ->setBucket($this->getBucketName()); } } diff --git a/src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php index 7923ab469c..83978d9764 100644 --- a/src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php +++ b/src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php @@ -18,6 +18,20 @@ final class PhabricatorFilesManagementMigrateWorkflow 'name' => 'dry-run', 'help' => pht('Show what would be migrated.'), ), + array( + 'name' => 'min-size', + 'param' => 'bytes', + 'help' => pht( + 'Do not migrate data for files which are smaller than a given '. + 'filesize.'), + ), + array( + 'name' => 'max-size', + 'param' => 'bytes', + 'help' => pht( + 'Do not migrate data for files which are larger than a given '. + 'filesize.'), + ), array( 'name' => 'all', 'help' => pht('Migrate all files.'), @@ -53,8 +67,13 @@ final class PhabricatorFilesManagementMigrateWorkflow $is_dry_run = $args->getArg('dry-run'); + $min_size = (int)$args->getArg('min-size'); + $max_size = (int)$args->getArg('max-size'); + $failed = array(); $engines = PhabricatorFileStorageEngine::loadAllEngines(); + $total_bytes = 0; + $total_files = 0; foreach ($iterator as $file) { $monogram = $file->getMonogram(); @@ -91,27 +110,59 @@ final class PhabricatorFilesManagementMigrateWorkflow continue; } + $byte_size = $file->getByteSize(); + + if ($min_size && ($byte_size < $min_size)) { + echo tsprintf( + "%s\n", + pht( + '%s: File size (%s) is smaller than minimum size (%s).', + $monogram, + phutil_format_bytes($byte_size), + phutil_format_bytes($min_size))); + continue; + } + + if ($max_size && ($byte_size > $max_size)) { + echo tsprintf( + "%s\n", + pht( + '%s: File size (%s) is larger than maximum size (%s).', + $monogram, + phutil_format_bytes($byte_size), + phutil_format_bytes($max_size))); + continue; + } + if ($is_dry_run) { echo tsprintf( "%s\n", pht( - '%s: Would migrate from "%s" to "%s" (dry run).', + '%s: (%s) Would migrate from "%s" to "%s" (dry run)...', $monogram, + phutil_format_bytes($byte_size), + $engine_key, + $target_key)); + } else { + echo tsprintf( + "%s\n", + pht( + '%s: (%s) Migrating from "%s" to "%s"...', + $monogram, + phutil_format_bytes($byte_size), $engine_key, $target_key)); - continue; } - echo tsprintf( - "%s\n", - pht( - '%s: Migrating from "%s" to "%s"...', - $monogram, - $engine_key, - $target_key)); - try { - $file->migrateToEngine($target_engine); + if ($is_dry_run) { + // Do nothing, this is a dry run. + } else { + $file->migrateToEngine($target_engine); + } + + $total_files += 1; + $total_bytes += $byte_size; echo tsprintf( "%s\n", @@ -127,6 +178,25 @@ final class PhabricatorFilesManagementMigrateWorkflow } } + echo tsprintf( + "%s\n", + pht( + 'Total Migrated Files: %s', + new PhutilNumber($total_files))); + + echo tsprintf( + "%s\n", + pht( + 'Total Migrated Bytes: %s', + phutil_format_bytes($total_bytes))); + + if ($is_dry_run) { + echo tsprintf( + "%s\n", + pht( + 'This was a dry run, so no real migrations were performed.')); + } + if ($failed) { $monograms = mpull($failed, 'getMonogram');