1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-10 00:42:41 +01:00

Add an Amazon S3 storage engine for Phabricator

Summary: Implements an S3 storage engine option for Phabricator.
Test Plan:
  - Uploaded files to S3.
  - Looked at them.
  - Verified they appeared in S3 using the S3 file browser.

Reviewed By: jungejason
Reviewers: jungejason, tuomaspelkonen, aran
CC: aran, jungejason
Differential Revision: 752
This commit is contained in:
epriestley 2011-07-31 13:54:58 -07:00
parent 07696e93fb
commit bd3a14f248
8 changed files with 2192 additions and 1 deletions

View file

@ -377,7 +377,14 @@ return array(
// disable it, specify null.
'storage.local-disk.path' => null,
// TODO: Implement S3.
// If you want to store files in Amazon S3, specify an AWS access and secret
// key here and a bucket name below.
'amazon-s3.access-key' => null,
'amazon-s3.secret-key' => null,
// Set this to a valid Amazon S3 bucket to store files there. You must also
// configure S3 access keys above.
'storage.s3.bucket' => null,
// Phabricator uses a storage engine selector to choose which storage engine
// to use when writing file data. If you add new storage engines or want to

105
externals/s3/README.txt vendored Normal file
View file

@ -0,0 +1,105 @@
AMAZON S3 PHP CLASS
USING THE CLASS
OO method (e,g; $s3->getObject(...)):
$s3 = new S3(awsAccessKey, awsSecretKey);
Statically (e,g; S3::getObject(...)):
S3::setAuth(awsAccessKey, awsSecretKey);
For class documentation see:
http://undesigned.org.za/files/s3-class-documentation/index.html
OBJECTS
Put an object from a string:
$s3->putObject($string, $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
Legacy function: $s3->putObjectString($string, $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
Put an object from a file:
$s3->putObject($s3->inputFile($file, false), $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
Legacy function: $s3->putObjectFile($uploadFile, $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
Put an object from a resource (buffer/file size is required):
Please note: the resource will be fclose()'d automatically
$s3->putObject($s3->inputResource(fopen($file, 'rb'), filesize($file)), $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
Get an object:
$s3->getObject($bucketName, $uploadName)
Save an object to file:
$s3->getObject($bucketName, $uploadName, $saveName)
Save an object to a resource of any type:
$s3->getObject($bucketName, $uploadName, fopen('savefile.txt', 'wb'))
Copy an object:
$s3->copyObject($srcBucket, $srcName, $bucketName, $saveName, $metaHeaders = array(), $requestHeaders = array())
Delete an object:
$s3->deleteObject($bucketName, $uploadName)
BUCKETS
Get a list of buckets:
$s3->listBuckets() // Simple bucket list
$s3->listBuckets(true) // Detailed bucket list
Create a public-read bucket:
$s3->putBucket($bucketName, S3::ACL_PUBLIC_READ)
$s3->putBucket($bucketName, S3::ACL_PUBLIC_READ, 'EU') // EU-hosted bucket
Get the contents of a bucket:
$s3->getBucket($bucketName)
Get a bucket's location:
$s3->getBucketLocation($bucketName)
Delete a bucket:
$s3->deleteBucket($bucketName)
KNOWN ISSUES
Files larger than 2GB are not supported on 32 bit systems due to PHPs signed integer problem
MORE INFORMATION
Project URL:
http://undesigned.org.za/2007/10/22/amazon-s3-php-class
Class documentation:
http://undesigned.org.za/files/s3-class-documentation/index.html
Bug reports:
https://github.com/tpyo/amazon-s3-php-class/issues
Amazon S3 documentation:
http://docs.amazonwebservices.com/AmazonS3/2006-03-01/
EOF

1923
externals/s3/S3.php vendored Normal file

File diff suppressed because it is too large Load diff

View file

@ -523,6 +523,7 @@ phutil_register_library_map(array(
'PhabricatorRepositorySvnCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/svn',
'PhabricatorRepositorySvnCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/svn',
'PhabricatorRepositoryType' => 'applications/repository/constants/repositorytype',
'PhabricatorS3FileStorageEngine' => 'applications/files/engine/s3',
'PhabricatorSQLPatchList' => 'infrastructure/setup/sql',
'PhabricatorSearchAbstractDocument' => 'applications/search/index/abstractdocument',
'PhabricatorSearchAttachController' => 'applications/search/controller/attach',
@ -1061,6 +1062,7 @@ phutil_register_library_map(array(
'PhabricatorRepositorySvnCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
'PhabricatorRepositorySvnCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
'PhabricatorRepositorySvnCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
'PhabricatorS3FileStorageEngine' => 'PhabricatorFileStorageEngine',
'PhabricatorSearchAttachController' => 'PhabricatorSearchController',
'PhabricatorSearchBaseController' => 'PhabricatorController',
'PhabricatorSearchCommitIndexer' => 'PhabricatorSearchDocumentIndexer',

View file

@ -0,0 +1,131 @@
<?php
/*
* Copyright 2011 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Amazon S3 file storage engine. This engine scales well but is relatively
* high-latency since data has to be pulled off S3.
*
* @task impl Implementation
* @task internal Internals
* @group filestorage
*/
final class PhabricatorS3FileStorageEngine
extends PhabricatorFileStorageEngine {
/* -( Implementation )----------------------------------------------------- */
/**
* This engine identifies as "amazon-s3".
*
* @task impl
*/
public function getEngineIdentifier() {
return 'amazon-s3';
}
/**
* Write file data into S3.
* @task impl
*/
public function writeFile($data, array $params) {
$s3 = $this->newS3API();
$name = 'phabricator/'.sha1(Filesystem::readRandomBytes(20));
$s3->putObject(
$data,
$this->getBucketName(),
$name,
$acl = 'private');
return $name;
}
/**
* Load a stored blob from S3.
* @task impl
*/
public function readFile($handle) {
$result = $this->newS3API()->getObject(
$this->getBucketName(),
$handle);
return $result->body;
}
/**
* Delete a blob from S3.
* @task impl
*/
public function deleteFile($handle) {
$this->newS3API()->deleteObject(
$this->getBucketName(),
$handle);
}
/* -( Internals )---------------------------------------------------------- */
/**
* Retrieve the S3 bucket name.
*
* @task internal
*/
private function getBucketName() {
$bucket = PhabricatorEnv::getEnvConfig('storage.s3.bucket');
if (!$bucket) {
throw new Exception("No 'storage.s3.bucket' specified!");
}
return $bucket;
}
/**
* Create a new S3 API object.
*
* @task internal
*/
private function newS3API() {
$libroot = dirname(phutil_get_library_root('phabricator'));
require_once $libroot.'/externals/s3/S3.php';
$access_key = PhabricatorEnv::getEnvConfig('amazon-s3.access-key');
$secret_key = PhabricatorEnv::getEnvConfig('amazon-s3.secret-key');
if (!$access_key || !$secret_key) {
throw new Exception(
"Specify 'amazon-s3.access-key' and 'amazon-s3.secret-key'!");
}
$s3 = newv(
'S3',
array(
$access_key,
$secret_key,
$use_ssl = true,
));
$s3->setExceptions(true);
return $s3;
}
}

View file

@ -0,0 +1,17 @@
<?php
/**
* This file is automatically generated. Lint this module to rebuild it.
* @generated
*/
phutil_require_module('phabricator', 'applications/files/engine/base');
phutil_require_module('phabricator', 'infrastructure/env');
phutil_require_module('phutil', 'filesystem');
phutil_require_module('phutil', 'moduleutils');
phutil_require_module('phutil', 'utils');
phutil_require_source('PhabricatorS3FileStorageEngine.php');

View file

@ -48,6 +48,11 @@ final class PhabricatorDefaultFileStorageEngineSelector
$engines[] = new PhabricatorLocalDiskFileStorageEngine();
}
$s3_key = 'storage.s3.bucket';
if (PhabricatorEnv::getEnvConfig($s3_key)) {
$engines[] = new PhabricatorS3FileStorageEngine();
}
if ($mysql_limit && empty($engines)) {
// If we return no engines, an exception will be thrown but it will be
// a little vague ("No valid storage engines"). Since this is a default

View file

@ -8,6 +8,7 @@
phutil_require_module('phabricator', 'applications/files/engine/localdisk');
phutil_require_module('phabricator', 'applications/files/engine/mysql');
phutil_require_module('phabricator', 'applications/files/engine/s3');
phutil_require_module('phabricator', 'applications/files/engineselector/base');
phutil_require_module('phabricator', 'infrastructure/env');