mirror of
https://we.phorge.it/source/phorge.git
synced 2024-12-23 22:10:55 +01:00
Add a script to migrate files between storage engines
Summary: Quora requested this (moving to S3) but it's also clearly a good idea. Test Plan: Ran with various valid/invalid options to test options. Error/sanity checking seemed OK. Migrated individual local files. Migrated all my local files back and forth between engines several times. Uploaded some new files. Reviewers: btrahan, vrana Reviewed By: vrana CC: aran Maniphest Tasks: T1950 Differential Revision: https://secure.phabricator.com/D3808
This commit is contained in:
parent
92a2866fb2
commit
5d1bd51627
8 changed files with 375 additions and 30 deletions
1
bin/files
Symbolic link
1
bin/files
Symbolic link
|
@ -0,0 +1 @@
|
|||
../scripts/files/manage_files.php
|
39
scripts/files/manage_files.php
Executable file
39
scripts/files/manage_files.php
Executable file
|
@ -0,0 +1,39 @@
|
|||
#!/usr/bin/env php
|
||||
<?php
|
||||
|
||||
/*
|
||||
* Copyright 2012 Facebook, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
$root = dirname(dirname(dirname(__FILE__)));
|
||||
require_once $root.'/scripts/__init_script__.php';
|
||||
|
||||
$args = new PhutilArgumentParser($argv);
|
||||
$args->setTagline('manage files');
|
||||
$args->setSynopsis(<<<EOSYNOPSIS
|
||||
**files** __command__ [__options__]
|
||||
Manage Phabricator file storage.
|
||||
|
||||
EOSYNOPSIS
|
||||
);
|
||||
$args->parseStandardArguments();
|
||||
|
||||
$workflows = array(
|
||||
new PhabricatorFilesManagementEnginesWorkflow(),
|
||||
new PhabricatorFilesManagementMigrateWorkflow(),
|
||||
new PhutilHelpArgumentWorkflow(),
|
||||
);
|
||||
|
||||
$args->parseWorkflows($workflows);
|
|
@ -755,6 +755,9 @@ phutil_register_library_map(array(
|
|||
'PhabricatorFileUploadController' => 'applications/files/controller/PhabricatorFileUploadController.php',
|
||||
'PhabricatorFileUploadException' => 'applications/files/exception/PhabricatorFileUploadException.php',
|
||||
'PhabricatorFileUploadView' => 'applications/files/view/PhabricatorFileUploadView.php',
|
||||
'PhabricatorFilesManagementEnginesWorkflow' => 'applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php',
|
||||
'PhabricatorFilesManagementMigrateWorkflow' => 'applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php',
|
||||
'PhabricatorFilesManagementWorkflow' => 'applications/files/management/PhabricatorFilesManagementWorkflow.php',
|
||||
'PhabricatorFlag' => 'applications/flag/storage/PhabricatorFlag.php',
|
||||
'PhabricatorFlagColor' => 'applications/flag/constants/PhabricatorFlagColor.php',
|
||||
'PhabricatorFlagConstants' => 'applications/flag/constants/PhabricatorFlagConstants.php',
|
||||
|
@ -1949,6 +1952,9 @@ phutil_register_library_map(array(
|
|||
'PhabricatorFileUploadController' => 'PhabricatorFileController',
|
||||
'PhabricatorFileUploadException' => 'Exception',
|
||||
'PhabricatorFileUploadView' => 'AphrontView',
|
||||
'PhabricatorFilesManagementEnginesWorkflow' => 'PhabricatorFilesManagementWorkflow',
|
||||
'PhabricatorFilesManagementMigrateWorkflow' => 'PhabricatorFilesManagementWorkflow',
|
||||
'PhabricatorFilesManagementWorkflow' => 'PhutilArgumentWorkflow',
|
||||
'PhabricatorFlag' => 'PhabricatorFlagDAO',
|
||||
'PhabricatorFlagColor' => 'PhabricatorFlagConstants',
|
||||
'PhabricatorFlagController' => 'PhabricatorController',
|
||||
|
|
|
@ -0,0 +1,46 @@
|
|||
<?php
|
||||
|
||||
/*
|
||||
* Copyright 2012 Facebook, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
final class PhabricatorFilesManagementEnginesWorkflow
|
||||
extends PhabricatorFilesManagementWorkflow {
|
||||
|
||||
public function didConstruct() {
|
||||
$this
|
||||
->setName('engines')
|
||||
->setSynopsis('List available storage engines.')
|
||||
->setArguments(array());
|
||||
}
|
||||
|
||||
public function execute(PhutilArgumentParser $args) {
|
||||
$console = PhutilConsole::getConsole();
|
||||
|
||||
$engines = PhabricatorFile::buildAllEngines();
|
||||
if (!$engines) {
|
||||
throw new Exception("No storage engines are available.");
|
||||
}
|
||||
|
||||
foreach ($engines as $engine) {
|
||||
$console->writeOut(
|
||||
"%s\n",
|
||||
$engine->getEngineIdentifier());
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,153 @@
|
|||
<?php
|
||||
|
||||
/*
|
||||
* Copyright 2012 Facebook, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
final class PhabricatorFilesManagementMigrateWorkflow
|
||||
extends PhabricatorFilesManagementWorkflow {
|
||||
|
||||
public function didConstruct() {
|
||||
$this
|
||||
->setName('migrate')
|
||||
->setSynopsis('Migrate files between storage engines.')
|
||||
->setArguments(
|
||||
array(
|
||||
array(
|
||||
'name' => 'engine',
|
||||
'param' => 'storage_engine',
|
||||
'help' => 'Migrate to the named storage engine.',
|
||||
),
|
||||
array(
|
||||
'name' => 'dry-run',
|
||||
'help' => 'Show what would be migrated.',
|
||||
),
|
||||
array(
|
||||
'name' => 'all',
|
||||
'help' => 'Migrate all files.',
|
||||
),
|
||||
array(
|
||||
'name' => 'names',
|
||||
'wildcard' => true,
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
public function execute(PhutilArgumentParser $args) {
|
||||
$console = PhutilConsole::getConsole();
|
||||
|
||||
$engine_id = $args->getArg('engine');
|
||||
if (!$engine_id) {
|
||||
throw new PhutilArgumentUsageException(
|
||||
"Specify an engine to migrate to with `--engine`. ".
|
||||
"Use `files engines` to get a list of engines.");
|
||||
}
|
||||
|
||||
$engine = PhabricatorFile::buildEngine($engine_id);
|
||||
|
||||
if ($args->getArg('all')) {
|
||||
if ($args->getArg('names')) {
|
||||
throw new PhutilArgumentUsageException(
|
||||
"Specify either a list of files or `--all`, but not both.");
|
||||
}
|
||||
$iterator = new LiskMigrationIterator(new PhabricatorFile());
|
||||
} else if ($args->getArg('names')) {
|
||||
$iterator = array();
|
||||
|
||||
foreach ($args->getArg('names') as $name) {
|
||||
$name = trim($name);
|
||||
|
||||
$id = preg_replace('/^F/i', '', $name);
|
||||
if (ctype_digit($id)) {
|
||||
$file = id(new PhabricatorFile())->loadOneWhere(
|
||||
'id = %d',
|
||||
$id);
|
||||
if (!$file) {
|
||||
throw new PhutilArgumentUsageException(
|
||||
"No file exists with id '{$name}'.");
|
||||
}
|
||||
} else {
|
||||
$file = id(new PhabricatorFile())->loadOneWhere(
|
||||
'phid = %d',
|
||||
$name);
|
||||
if (!$file) {
|
||||
throw new PhutilArgumentUsageException(
|
||||
"No file exists with PHID '{$name}'.");
|
||||
}
|
||||
}
|
||||
$iterator[] = $file;
|
||||
}
|
||||
} else {
|
||||
throw new PhutilArgumentUsageException(
|
||||
"Either specify a list of files to migrate, or use `--all` ".
|
||||
"to migrate all files.");
|
||||
}
|
||||
|
||||
$is_dry_run = $args->getArg('dry-run');
|
||||
|
||||
$failed = array();
|
||||
|
||||
foreach ($iterator as $file) {
|
||||
$fid = 'F'.$file->getID();
|
||||
|
||||
if ($file->getStorageEngine() == $engine_id) {
|
||||
$console->writeOut(
|
||||
"%s: Already stored on '%s'\n",
|
||||
$fid,
|
||||
$engine_id);
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($is_dry_run) {
|
||||
$console->writeOut(
|
||||
"%s: Would migrate from '%s' to '%s' (dry run)\n",
|
||||
$fid,
|
||||
$file->getStorageEngine(),
|
||||
$engine_id);
|
||||
continue;
|
||||
}
|
||||
|
||||
$console->writeOut(
|
||||
"%s: Migrating from '%s' to '%s'...",
|
||||
$fid,
|
||||
$file->getStorageEngine(),
|
||||
$engine_id);
|
||||
|
||||
try {
|
||||
$file->migrateToEngine($engine);
|
||||
$console->writeOut("done.\n");
|
||||
} catch (Exception $ex) {
|
||||
$console->writeOut("failed!\n");
|
||||
$console->writeErr("%s\n", (string)$ex);
|
||||
$failed[] = $file;
|
||||
}
|
||||
}
|
||||
|
||||
if ($failed) {
|
||||
$console->writeOut("**Failures!**\n");
|
||||
$ids = array();
|
||||
foreach ($failed as $file) {
|
||||
$ids[] = 'F'.$file->getID();
|
||||
}
|
||||
$console->writeOut("%s\n", implode(', ', $ids));
|
||||
|
||||
return 1;
|
||||
} else {
|
||||
$console->writeOut("**Success!**\n");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,26 @@
|
|||
<?php
|
||||
|
||||
/*
|
||||
* Copyright 2012 Facebook, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
abstract class PhabricatorFilesManagementWorkflow
|
||||
extends PhutilArgumentWorkflow {
|
||||
|
||||
public function isExecutable() {
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
|
@ -149,44 +149,31 @@ final class PhabricatorFile extends PhabricatorFileDAO {
|
|||
throw new Exception("No valid storage engines are available!");
|
||||
}
|
||||
|
||||
$file = new PhabricatorFile();
|
||||
|
||||
$data_handle = null;
|
||||
$engine_identifier = null;
|
||||
$exceptions = array();
|
||||
foreach ($engines as $engine) {
|
||||
$engine_class = get_class($engine);
|
||||
try {
|
||||
// Perform the actual write.
|
||||
$data_handle = $engine->writeFile($data, $params);
|
||||
if (!$data_handle || strlen($data_handle) > 255) {
|
||||
// This indicates an improperly implemented storage engine.
|
||||
throw new PhabricatorFileStorageConfigurationException(
|
||||
"Storage engine '{$engine_class}' executed writeFile() but did ".
|
||||
"not return a valid handle ('{$data_handle}') to the data: it ".
|
||||
"must be nonempty and no longer than 255 characters.");
|
||||
}
|
||||
|
||||
$engine_identifier = $engine->getEngineIdentifier();
|
||||
if (!$engine_identifier || strlen($engine_identifier) > 32) {
|
||||
throw new PhabricatorFileStorageConfigurationException(
|
||||
"Storage engine '{$engine_class}' returned an improper engine ".
|
||||
"identifier '{$engine_identifier}': it must be nonempty ".
|
||||
"and no longer than 32 characters.");
|
||||
}
|
||||
list($engine_identifier, $data_handle) = $file->writeToEngine(
|
||||
$engine,
|
||||
$data,
|
||||
$params);
|
||||
|
||||
// We stored the file somewhere so stop trying to write it to other
|
||||
// places.
|
||||
break;
|
||||
|
||||
} catch (PhabricatorFileStorageConfigurationException $ex) {
|
||||
// If an engine is outright misconfigured (or misimplemented), raise
|
||||
// that immediately since it probably needs attention.
|
||||
throw $ex;
|
||||
|
||||
} catch (Exception $ex) {
|
||||
// If an engine doesn't work, keep trying all the other valid engines
|
||||
// in case something else works.
|
||||
phlog($ex);
|
||||
|
||||
// If an engine doesn't work, keep trying all the other valid engines
|
||||
// in case something else works.
|
||||
$exceptions[$engine_class] = $ex;
|
||||
}
|
||||
}
|
||||
|
@ -204,7 +191,6 @@ final class PhabricatorFile extends PhabricatorFileDAO {
|
|||
// (always the case with newFromFileDownload()), store a ''
|
||||
$authorPHID = idx($params, 'authorPHID');
|
||||
|
||||
$file = new PhabricatorFile();
|
||||
$file->setName($file_name);
|
||||
$file->setByteSize(strlen($data));
|
||||
$file->setAuthorPHID($authorPHID);
|
||||
|
@ -230,6 +216,63 @@ final class PhabricatorFile extends PhabricatorFileDAO {
|
|||
return $file;
|
||||
}
|
||||
|
||||
public function migrateToEngine(PhabricatorFileStorageEngine $engine) {
|
||||
if (!$this->getID() || !$this->getStorageHandle()) {
|
||||
throw new Exception(
|
||||
"You can not migrate a file which hasn't yet been saved.");
|
||||
}
|
||||
|
||||
$data = $this->loadFileData();
|
||||
$params = array(
|
||||
'name' => $this->getName(),
|
||||
);
|
||||
|
||||
list($new_identifier, $new_handle) = $this->writeToEngine(
|
||||
$engine,
|
||||
$data,
|
||||
$params);
|
||||
|
||||
$old_engine = $this->instantiateStorageEngine();
|
||||
$old_handle = $this->getStorageHandle();
|
||||
|
||||
$this->setStorageEngine($new_identifier);
|
||||
$this->setStorageHandle($new_handle);
|
||||
$this->save();
|
||||
|
||||
$old_engine->deleteFile($old_handle);
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
private function writeToEngine(
|
||||
PhabricatorFileStorageEngine $engine,
|
||||
$data,
|
||||
array $params) {
|
||||
|
||||
$engine_class = get_class($engine);
|
||||
|
||||
$data_handle = $engine->writeFile($data, $params);
|
||||
|
||||
if (!$data_handle || strlen($data_handle) > 255) {
|
||||
// This indicates an improperly implemented storage engine.
|
||||
throw new PhabricatorFileStorageConfigurationException(
|
||||
"Storage engine '{$engine_class}' executed writeFile() but did ".
|
||||
"not return a valid handle ('{$data_handle}') to the data: it ".
|
||||
"must be nonempty and no longer than 255 characters.");
|
||||
}
|
||||
|
||||
$engine_identifier = $engine->getEngineIdentifier();
|
||||
if (!$engine_identifier || strlen($engine_identifier) > 32) {
|
||||
throw new PhabricatorFileStorageConfigurationException(
|
||||
"Storage engine '{$engine_class}' returned an improper engine ".
|
||||
"identifier '{$engine_identifier}': it must be nonempty ".
|
||||
"and no longer than 32 characters.");
|
||||
}
|
||||
|
||||
return array($engine_identifier, $data_handle);
|
||||
}
|
||||
|
||||
|
||||
public static function newFromFileDownload($uri, $name) {
|
||||
$uri = new PhutilURI($uri);
|
||||
|
||||
|
@ -402,19 +445,34 @@ final class PhabricatorFile extends PhabricatorFileDAO {
|
|||
}
|
||||
|
||||
protected function instantiateStorageEngine() {
|
||||
$engines = id(new PhutilSymbolLoader())
|
||||
->setType('class')
|
||||
->setAncestorClass('PhabricatorFileStorageEngine')
|
||||
->selectAndLoadSymbols();
|
||||
return self::buildEngine($this->getStorageEngine());
|
||||
}
|
||||
|
||||
foreach ($engines as $engine_class) {
|
||||
$engine = newv($engine_class['name'], array());
|
||||
if ($engine->getEngineIdentifier() == $this->getStorageEngine()) {
|
||||
public static function buildEngine($engine_identifier) {
|
||||
$engines = self::buildAllEngines();
|
||||
foreach ($engines as $engine) {
|
||||
if ($engine->getEngineIdentifier() == $engine_identifier) {
|
||||
return $engine;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Exception("File's storage engine could be located!");
|
||||
throw new Exception(
|
||||
"Storage engine '{$engine_identifier}' could not be located!");
|
||||
}
|
||||
|
||||
public static function buildAllEngines() {
|
||||
$engines = id(new PhutilSymbolLoader())
|
||||
->setType('class')
|
||||
->setConcreteOnly(true)
|
||||
->setAncestorClass('PhabricatorFileStorageEngine')
|
||||
->selectAndLoadSymbols();
|
||||
|
||||
$results = array();
|
||||
foreach ($engines as $engine_class) {
|
||||
$results[] = newv($engine_class['name'], array());
|
||||
}
|
||||
|
||||
return $results;
|
||||
}
|
||||
|
||||
public function getViewableMimeType() {
|
||||
|
|
|
@ -80,6 +80,22 @@ Technical Documentation}.
|
|||
You can test that things are correctly configured by going to the Files
|
||||
application (##/file/##) and uploading files.
|
||||
|
||||
= Migrating Files Beteeen Engines =
|
||||
|
||||
If you want to move files between storage engines, you can use the `bin/files`
|
||||
script to perform migrations. For example, suppose you previously used MySQL but
|
||||
recently set up S3 and want to migrate all your files there. First, migrate one
|
||||
file to make sure things work:
|
||||
|
||||
phabricator/ $ ./bin/files migrate --engine amazon-s3 F12345
|
||||
|
||||
If that works properly, you can then migrate everything:
|
||||
|
||||
phabricator/ $ ./bin/files migrate --engine amazon-s3 --all
|
||||
|
||||
You can use `--dry-run` to show which migrations would be performed without
|
||||
taking any action. Run `bin/files help` for more options and information.
|
||||
|
||||
= Next Steps =
|
||||
|
||||
Continue by:
|
||||
|
|
Loading…
Reference in a new issue