2011-07-31 22:54:58 +02:00
|
|
|
<?php
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Amazon S3 file storage engine. This engine scales well but is relatively
|
|
|
|
* high-latency since data has to be pulled off S3.
|
|
|
|
*
|
|
|
|
* @task impl Implementation
|
|
|
|
* @task internal Internals
|
|
|
|
* @group filestorage
|
|
|
|
*/
|
|
|
|
final class PhabricatorS3FileStorageEngine
|
|
|
|
extends PhabricatorFileStorageEngine {
|
|
|
|
|
|
|
|
/* -( Implementation )----------------------------------------------------- */
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* This engine identifies as "amazon-s3".
|
|
|
|
*
|
|
|
|
* @task impl
|
|
|
|
*/
|
|
|
|
public function getEngineIdentifier() {
|
|
|
|
return 'amazon-s3';
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Write file data into S3.
|
|
|
|
* @task impl
|
|
|
|
*/
|
|
|
|
public function writeFile($data, array $params) {
|
|
|
|
$s3 = $this->newS3API();
|
|
|
|
|
2012-12-30 22:44:37 +01:00
|
|
|
// Generate a random name for this file. We add some directories to it
|
|
|
|
// (e.g. 'abcdef123456' becomes 'ab/cd/ef123456') to make large numbers of
|
|
|
|
// files more browsable with web/debugging tools like the S3 administration
|
|
|
|
// tool.
|
|
|
|
$seed = Filesystem::readRandomCharacters(20);
|
|
|
|
$parts = array(
|
|
|
|
substr($seed, 0, 2),
|
|
|
|
substr($seed, 2, 2),
|
|
|
|
substr($seed, 4),
|
|
|
|
);
|
|
|
|
$name = 'phabricator/'.implode('/', $parts);
|
2011-07-31 22:54:58 +02:00
|
|
|
|
Create AphrontWriteGuard, a backup mechanism for CSRF validation
Summary:
Provide a catchall mechanism to find unprotected writes.
- Depends on D758.
- Similar to WriteOnHTTPGet stuff from Facebook's stack.
- Since we have a small number of storage mechanisms and highly structured
read/write pathways, we can explicitly answer the question "is this page
performing a write?".
- Never allow writes without CSRF checks.
- This will probably break some things. That's fine: they're CSRF
vulnerabilities or weird edge cases that we can fix. But don't push to Facebook
for a few days unless you're prepared to deal with this.
- **>>> MEGADERP: All Conduit write APIs are currently vulnerable to CSRF!
<<<**
Test Plan:
- Ran some scripts that perform writes (scripts/search indexers), no issues.
- Performed normal CSRF submits.
- Added writes to an un-CSRF'd page, got an exception.
- Executed conduit methods.
- Did login/logout (this works because the logged-out user validates the
logged-out csrf "token").
- Did OAuth login.
- Did OAuth registration.
Reviewers: pedram, andrewjcg, erling, jungejason, tuomaspelkonen, aran,
codeblock
Commenters: pedram
CC: aran, epriestley, pedram
Differential Revision: 777
2011-08-03 20:49:27 +02:00
|
|
|
AphrontWriteGuard::willWrite();
|
2011-07-31 22:54:58 +02:00
|
|
|
$s3->putObject(
|
|
|
|
$data,
|
|
|
|
$this->getBucketName(),
|
|
|
|
$name,
|
|
|
|
$acl = 'private');
|
|
|
|
|
|
|
|
return $name;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Load a stored blob from S3.
|
|
|
|
* @task impl
|
|
|
|
*/
|
|
|
|
public function readFile($handle) {
|
|
|
|
$result = $this->newS3API()->getObject(
|
|
|
|
$this->getBucketName(),
|
|
|
|
$handle);
|
2012-12-30 22:44:37 +01:00
|
|
|
|
|
|
|
// NOTE: The implementation of the API that we're using may respond with
|
|
|
|
// a successful result that has length 0 and no body property.
|
|
|
|
if (isset($result->body)) {
|
|
|
|
return $result->body;
|
|
|
|
} else {
|
|
|
|
return '';
|
|
|
|
}
|
2011-07-31 22:54:58 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Delete a blob from S3.
|
|
|
|
* @task impl
|
|
|
|
*/
|
|
|
|
public function deleteFile($handle) {
|
Create AphrontWriteGuard, a backup mechanism for CSRF validation
Summary:
Provide a catchall mechanism to find unprotected writes.
- Depends on D758.
- Similar to WriteOnHTTPGet stuff from Facebook's stack.
- Since we have a small number of storage mechanisms and highly structured
read/write pathways, we can explicitly answer the question "is this page
performing a write?".
- Never allow writes without CSRF checks.
- This will probably break some things. That's fine: they're CSRF
vulnerabilities or weird edge cases that we can fix. But don't push to Facebook
for a few days unless you're prepared to deal with this.
- **>>> MEGADERP: All Conduit write APIs are currently vulnerable to CSRF!
<<<**
Test Plan:
- Ran some scripts that perform writes (scripts/search indexers), no issues.
- Performed normal CSRF submits.
- Added writes to an un-CSRF'd page, got an exception.
- Executed conduit methods.
- Did login/logout (this works because the logged-out user validates the
logged-out csrf "token").
- Did OAuth login.
- Did OAuth registration.
Reviewers: pedram, andrewjcg, erling, jungejason, tuomaspelkonen, aran,
codeblock
Commenters: pedram
CC: aran, epriestley, pedram
Differential Revision: 777
2011-08-03 20:49:27 +02:00
|
|
|
AphrontWriteGuard::willWrite();
|
2011-07-31 22:54:58 +02:00
|
|
|
$this->newS3API()->deleteObject(
|
|
|
|
$this->getBucketName(),
|
|
|
|
$handle);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/* -( Internals )---------------------------------------------------------- */
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Retrieve the S3 bucket name.
|
|
|
|
*
|
|
|
|
* @task internal
|
|
|
|
*/
|
|
|
|
private function getBucketName() {
|
|
|
|
$bucket = PhabricatorEnv::getEnvConfig('storage.s3.bucket');
|
|
|
|
if (!$bucket) {
|
2012-04-09 00:07:34 +02:00
|
|
|
throw new PhabricatorFileStorageConfigurationException(
|
|
|
|
"No 'storage.s3.bucket' specified!");
|
2011-07-31 22:54:58 +02:00
|
|
|
}
|
|
|
|
return $bucket;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Create a new S3 API object.
|
|
|
|
*
|
|
|
|
* @task internal
|
2012-05-31 01:38:53 +02:00
|
|
|
* @phutil-external-symbol class S3
|
2011-07-31 22:54:58 +02:00
|
|
|
*/
|
|
|
|
private function newS3API() {
|
|
|
|
$libroot = dirname(phutil_get_library_root('phabricator'));
|
|
|
|
require_once $libroot.'/externals/s3/S3.php';
|
|
|
|
|
|
|
|
$access_key = PhabricatorEnv::getEnvConfig('amazon-s3.access-key');
|
|
|
|
$secret_key = PhabricatorEnv::getEnvConfig('amazon-s3.secret-key');
|
2012-11-16 13:07:57 +01:00
|
|
|
$endpoint = PhabricatorEnv::getEnvConfig('amazon-s3.endpoint');
|
2011-07-31 22:54:58 +02:00
|
|
|
|
|
|
|
if (!$access_key || !$secret_key) {
|
2012-04-09 00:07:34 +02:00
|
|
|
throw new PhabricatorFileStorageConfigurationException(
|
2011-07-31 22:54:58 +02:00
|
|
|
"Specify 'amazon-s3.access-key' and 'amazon-s3.secret-key'!");
|
|
|
|
}
|
|
|
|
|
2012-11-16 13:07:57 +01:00
|
|
|
if ($endpoint !== null) {
|
|
|
|
$s3 = new S3($access_key, $secret_key, $use_ssl = true, $endpoint);
|
|
|
|
} else {
|
|
|
|
$s3 = new S3($access_key, $secret_key, $use_ssl = true);
|
|
|
|
}
|
2011-07-31 22:54:58 +02:00
|
|
|
|
|
|
|
$s3->setExceptions(true);
|
|
|
|
|
|
|
|
return $s3;
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|