mirror of
https://we.phorge.it/source/phorge.git
synced 2024-11-10 00:42:41 +01:00
Add an Amazon S3 storage engine for Phabricator
Summary: Implements an S3 storage engine option for Phabricator. Test Plan: - Uploaded files to S3. - Looked at them. - Verified they appeared in S3 using the S3 file browser. Reviewed By: jungejason Reviewers: jungejason, tuomaspelkonen, aran CC: aran, jungejason Differential Revision: 752
This commit is contained in:
parent
07696e93fb
commit
bd3a14f248
8 changed files with 2192 additions and 1 deletions
|
@ -377,7 +377,14 @@ return array(
|
|||
// disable it, specify null.
|
||||
'storage.local-disk.path' => null,
|
||||
|
||||
// TODO: Implement S3.
|
||||
// If you want to store files in Amazon S3, specify an AWS access and secret
|
||||
// key here and a bucket name below.
|
||||
'amazon-s3.access-key' => null,
|
||||
'amazon-s3.secret-key' => null,
|
||||
|
||||
// Set this to a valid Amazon S3 bucket to store files there. You must also
|
||||
// configure S3 access keys above.
|
||||
'storage.s3.bucket' => null,
|
||||
|
||||
// Phabricator uses a storage engine selector to choose which storage engine
|
||||
// to use when writing file data. If you add new storage engines or want to
|
||||
|
|
105
externals/s3/README.txt
vendored
Normal file
105
externals/s3/README.txt
vendored
Normal file
|
@ -0,0 +1,105 @@
|
|||
AMAZON S3 PHP CLASS
|
||||
|
||||
|
||||
USING THE CLASS
|
||||
|
||||
OO method (e,g; $s3->getObject(...)):
|
||||
$s3 = new S3(awsAccessKey, awsSecretKey);
|
||||
|
||||
Statically (e,g; S3::getObject(...)):
|
||||
S3::setAuth(awsAccessKey, awsSecretKey);
|
||||
|
||||
|
||||
For class documentation see:
|
||||
http://undesigned.org.za/files/s3-class-documentation/index.html
|
||||
|
||||
|
||||
OBJECTS
|
||||
|
||||
|
||||
Put an object from a string:
|
||||
$s3->putObject($string, $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
|
||||
Legacy function: $s3->putObjectString($string, $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
|
||||
|
||||
|
||||
Put an object from a file:
|
||||
$s3->putObject($s3->inputFile($file, false), $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
|
||||
Legacy function: $s3->putObjectFile($uploadFile, $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
|
||||
|
||||
|
||||
Put an object from a resource (buffer/file size is required):
|
||||
Please note: the resource will be fclose()'d automatically
|
||||
$s3->putObject($s3->inputResource(fopen($file, 'rb'), filesize($file)), $bucketName, $uploadName, S3::ACL_PUBLIC_READ)
|
||||
|
||||
|
||||
Get an object:
|
||||
$s3->getObject($bucketName, $uploadName)
|
||||
|
||||
|
||||
Save an object to file:
|
||||
$s3->getObject($bucketName, $uploadName, $saveName)
|
||||
|
||||
|
||||
Save an object to a resource of any type:
|
||||
$s3->getObject($bucketName, $uploadName, fopen('savefile.txt', 'wb'))
|
||||
|
||||
|
||||
Copy an object:
|
||||
$s3->copyObject($srcBucket, $srcName, $bucketName, $saveName, $metaHeaders = array(), $requestHeaders = array())
|
||||
|
||||
|
||||
Delete an object:
|
||||
$s3->deleteObject($bucketName, $uploadName)
|
||||
|
||||
|
||||
|
||||
BUCKETS
|
||||
|
||||
|
||||
Get a list of buckets:
|
||||
$s3->listBuckets() // Simple bucket list
|
||||
$s3->listBuckets(true) // Detailed bucket list
|
||||
|
||||
|
||||
Create a public-read bucket:
|
||||
$s3->putBucket($bucketName, S3::ACL_PUBLIC_READ)
|
||||
$s3->putBucket($bucketName, S3::ACL_PUBLIC_READ, 'EU') // EU-hosted bucket
|
||||
|
||||
|
||||
Get the contents of a bucket:
|
||||
$s3->getBucket($bucketName)
|
||||
|
||||
|
||||
Get a bucket's location:
|
||||
$s3->getBucketLocation($bucketName)
|
||||
|
||||
|
||||
Delete a bucket:
|
||||
$s3->deleteBucket($bucketName)
|
||||
|
||||
|
||||
|
||||
|
||||
KNOWN ISSUES
|
||||
|
||||
Files larger than 2GB are not supported on 32 bit systems due to PHP’s signed integer problem
|
||||
|
||||
|
||||
|
||||
MORE INFORMATION
|
||||
|
||||
|
||||
Project URL:
|
||||
http://undesigned.org.za/2007/10/22/amazon-s3-php-class
|
||||
|
||||
Class documentation:
|
||||
http://undesigned.org.za/files/s3-class-documentation/index.html
|
||||
|
||||
Bug reports:
|
||||
https://github.com/tpyo/amazon-s3-php-class/issues
|
||||
|
||||
Amazon S3 documentation:
|
||||
http://docs.amazonwebservices.com/AmazonS3/2006-03-01/
|
||||
|
||||
|
||||
EOF
|
1923
externals/s3/S3.php
vendored
Normal file
1923
externals/s3/S3.php
vendored
Normal file
File diff suppressed because it is too large
Load diff
|
@ -523,6 +523,7 @@ phutil_register_library_map(array(
|
|||
'PhabricatorRepositorySvnCommitDiscoveryDaemon' => 'applications/repository/daemon/commitdiscovery/svn',
|
||||
'PhabricatorRepositorySvnCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/svn',
|
||||
'PhabricatorRepositoryType' => 'applications/repository/constants/repositorytype',
|
||||
'PhabricatorS3FileStorageEngine' => 'applications/files/engine/s3',
|
||||
'PhabricatorSQLPatchList' => 'infrastructure/setup/sql',
|
||||
'PhabricatorSearchAbstractDocument' => 'applications/search/index/abstractdocument',
|
||||
'PhabricatorSearchAttachController' => 'applications/search/controller/attach',
|
||||
|
@ -1061,6 +1062,7 @@ phutil_register_library_map(array(
|
|||
'PhabricatorRepositorySvnCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
|
||||
'PhabricatorRepositorySvnCommitDiscoveryDaemon' => 'PhabricatorRepositoryCommitDiscoveryDaemon',
|
||||
'PhabricatorRepositorySvnCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
|
||||
'PhabricatorS3FileStorageEngine' => 'PhabricatorFileStorageEngine',
|
||||
'PhabricatorSearchAttachController' => 'PhabricatorSearchController',
|
||||
'PhabricatorSearchBaseController' => 'PhabricatorController',
|
||||
'PhabricatorSearchCommitIndexer' => 'PhabricatorSearchDocumentIndexer',
|
||||
|
|
|
@ -0,0 +1,131 @@
|
|||
<?php
|
||||
|
||||
/*
|
||||
* Copyright 2011 Facebook, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Amazon S3 file storage engine. This engine scales well but is relatively
|
||||
* high-latency since data has to be pulled off S3.
|
||||
*
|
||||
* @task impl Implementation
|
||||
* @task internal Internals
|
||||
* @group filestorage
|
||||
*/
|
||||
final class PhabricatorS3FileStorageEngine
|
||||
extends PhabricatorFileStorageEngine {
|
||||
|
||||
/* -( Implementation )----------------------------------------------------- */
|
||||
|
||||
|
||||
/**
|
||||
* This engine identifies as "amazon-s3".
|
||||
*
|
||||
* @task impl
|
||||
*/
|
||||
public function getEngineIdentifier() {
|
||||
return 'amazon-s3';
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Write file data into S3.
|
||||
* @task impl
|
||||
*/
|
||||
public function writeFile($data, array $params) {
|
||||
$s3 = $this->newS3API();
|
||||
|
||||
$name = 'phabricator/'.sha1(Filesystem::readRandomBytes(20));
|
||||
|
||||
$s3->putObject(
|
||||
$data,
|
||||
$this->getBucketName(),
|
||||
$name,
|
||||
$acl = 'private');
|
||||
|
||||
return $name;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Load a stored blob from S3.
|
||||
* @task impl
|
||||
*/
|
||||
public function readFile($handle) {
|
||||
$result = $this->newS3API()->getObject(
|
||||
$this->getBucketName(),
|
||||
$handle);
|
||||
return $result->body;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Delete a blob from S3.
|
||||
* @task impl
|
||||
*/
|
||||
public function deleteFile($handle) {
|
||||
$this->newS3API()->deleteObject(
|
||||
$this->getBucketName(),
|
||||
$handle);
|
||||
}
|
||||
|
||||
|
||||
/* -( Internals )---------------------------------------------------------- */
|
||||
|
||||
|
||||
/**
|
||||
* Retrieve the S3 bucket name.
|
||||
*
|
||||
* @task internal
|
||||
*/
|
||||
private function getBucketName() {
|
||||
$bucket = PhabricatorEnv::getEnvConfig('storage.s3.bucket');
|
||||
if (!$bucket) {
|
||||
throw new Exception("No 'storage.s3.bucket' specified!");
|
||||
}
|
||||
return $bucket;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new S3 API object.
|
||||
*
|
||||
* @task internal
|
||||
*/
|
||||
private function newS3API() {
|
||||
$libroot = dirname(phutil_get_library_root('phabricator'));
|
||||
require_once $libroot.'/externals/s3/S3.php';
|
||||
|
||||
$access_key = PhabricatorEnv::getEnvConfig('amazon-s3.access-key');
|
||||
$secret_key = PhabricatorEnv::getEnvConfig('amazon-s3.secret-key');
|
||||
|
||||
if (!$access_key || !$secret_key) {
|
||||
throw new Exception(
|
||||
"Specify 'amazon-s3.access-key' and 'amazon-s3.secret-key'!");
|
||||
}
|
||||
|
||||
$s3 = newv(
|
||||
'S3',
|
||||
array(
|
||||
$access_key,
|
||||
$secret_key,
|
||||
$use_ssl = true,
|
||||
));
|
||||
|
||||
$s3->setExceptions(true);
|
||||
|
||||
return $s3;
|
||||
}
|
||||
|
||||
}
|
17
src/applications/files/engine/s3/__init__.php
Normal file
17
src/applications/files/engine/s3/__init__.php
Normal file
|
@ -0,0 +1,17 @@
|
|||
<?php
|
||||
/**
|
||||
* This file is automatically generated. Lint this module to rebuild it.
|
||||
* @generated
|
||||
*/
|
||||
|
||||
|
||||
|
||||
phutil_require_module('phabricator', 'applications/files/engine/base');
|
||||
phutil_require_module('phabricator', 'infrastructure/env');
|
||||
|
||||
phutil_require_module('phutil', 'filesystem');
|
||||
phutil_require_module('phutil', 'moduleutils');
|
||||
phutil_require_module('phutil', 'utils');
|
||||
|
||||
|
||||
phutil_require_source('PhabricatorS3FileStorageEngine.php');
|
|
@ -48,6 +48,11 @@ final class PhabricatorDefaultFileStorageEngineSelector
|
|||
$engines[] = new PhabricatorLocalDiskFileStorageEngine();
|
||||
}
|
||||
|
||||
$s3_key = 'storage.s3.bucket';
|
||||
if (PhabricatorEnv::getEnvConfig($s3_key)) {
|
||||
$engines[] = new PhabricatorS3FileStorageEngine();
|
||||
}
|
||||
|
||||
if ($mysql_limit && empty($engines)) {
|
||||
// If we return no engines, an exception will be thrown but it will be
|
||||
// a little vague ("No valid storage engines"). Since this is a default
|
||||
|
|
|
@ -8,6 +8,7 @@
|
|||
|
||||
phutil_require_module('phabricator', 'applications/files/engine/localdisk');
|
||||
phutil_require_module('phabricator', 'applications/files/engine/mysql');
|
||||
phutil_require_module('phabricator', 'applications/files/engine/s3');
|
||||
phutil_require_module('phabricator', 'applications/files/engineselector/base');
|
||||
phutil_require_module('phabricator', 'infrastructure/env');
|
||||
|
||||
|
|
Loading…
Reference in a new issue