1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-09 16:32:39 +01:00

Add a script to migrate files between storage engines

Summary: Quora requested this (moving to S3) but it's also clearly a good idea.

Test Plan:
Ran with various valid/invalid options to test options. Error/sanity checking seemed OK.

Migrated individual local files.

Migrated all my local files back and forth between engines several times.

Uploaded some new files.

Reviewers: btrahan, vrana

Reviewed By: vrana

CC: aran

Maniphest Tasks: T1950

Differential Revision: https://secure.phabricator.com/D3808
This commit is contained in:
epriestley 2012-10-25 11:36:38 -07:00
parent 92a2866fb2
commit 5d1bd51627
8 changed files with 375 additions and 30 deletions

1
bin/files Symbolic link
View file

@ -0,0 +1 @@
../scripts/files/manage_files.php

39
scripts/files/manage_files.php Executable file
View file

@ -0,0 +1,39 @@
#!/usr/bin/env php
<?php
/*
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
$root = dirname(dirname(dirname(__FILE__)));
require_once $root.'/scripts/__init_script__.php';
$args = new PhutilArgumentParser($argv);
$args->setTagline('manage files');
$args->setSynopsis(<<<EOSYNOPSIS
**files** __command__ [__options__]
Manage Phabricator file storage.
EOSYNOPSIS
);
$args->parseStandardArguments();
$workflows = array(
new PhabricatorFilesManagementEnginesWorkflow(),
new PhabricatorFilesManagementMigrateWorkflow(),
new PhutilHelpArgumentWorkflow(),
);
$args->parseWorkflows($workflows);

View file

@ -755,6 +755,9 @@ phutil_register_library_map(array(
'PhabricatorFileUploadController' => 'applications/files/controller/PhabricatorFileUploadController.php',
'PhabricatorFileUploadException' => 'applications/files/exception/PhabricatorFileUploadException.php',
'PhabricatorFileUploadView' => 'applications/files/view/PhabricatorFileUploadView.php',
'PhabricatorFilesManagementEnginesWorkflow' => 'applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php',
'PhabricatorFilesManagementMigrateWorkflow' => 'applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php',
'PhabricatorFilesManagementWorkflow' => 'applications/files/management/PhabricatorFilesManagementWorkflow.php',
'PhabricatorFlag' => 'applications/flag/storage/PhabricatorFlag.php',
'PhabricatorFlagColor' => 'applications/flag/constants/PhabricatorFlagColor.php',
'PhabricatorFlagConstants' => 'applications/flag/constants/PhabricatorFlagConstants.php',
@ -1949,6 +1952,9 @@ phutil_register_library_map(array(
'PhabricatorFileUploadController' => 'PhabricatorFileController',
'PhabricatorFileUploadException' => 'Exception',
'PhabricatorFileUploadView' => 'AphrontView',
'PhabricatorFilesManagementEnginesWorkflow' => 'PhabricatorFilesManagementWorkflow',
'PhabricatorFilesManagementMigrateWorkflow' => 'PhabricatorFilesManagementWorkflow',
'PhabricatorFilesManagementWorkflow' => 'PhutilArgumentWorkflow',
'PhabricatorFlag' => 'PhabricatorFlagDAO',
'PhabricatorFlagColor' => 'PhabricatorFlagConstants',
'PhabricatorFlagController' => 'PhabricatorController',

View file

@ -0,0 +1,46 @@
<?php
/*
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
final class PhabricatorFilesManagementEnginesWorkflow
extends PhabricatorFilesManagementWorkflow {
public function didConstruct() {
$this
->setName('engines')
->setSynopsis('List available storage engines.')
->setArguments(array());
}
public function execute(PhutilArgumentParser $args) {
$console = PhutilConsole::getConsole();
$engines = PhabricatorFile::buildAllEngines();
if (!$engines) {
throw new Exception("No storage engines are available.");
}
foreach ($engines as $engine) {
$console->writeOut(
"%s\n",
$engine->getEngineIdentifier());
}
return 0;
}
}

View file

@ -0,0 +1,153 @@
<?php
/*
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
final class PhabricatorFilesManagementMigrateWorkflow
extends PhabricatorFilesManagementWorkflow {
public function didConstruct() {
$this
->setName('migrate')
->setSynopsis('Migrate files between storage engines.')
->setArguments(
array(
array(
'name' => 'engine',
'param' => 'storage_engine',
'help' => 'Migrate to the named storage engine.',
),
array(
'name' => 'dry-run',
'help' => 'Show what would be migrated.',
),
array(
'name' => 'all',
'help' => 'Migrate all files.',
),
array(
'name' => 'names',
'wildcard' => true,
),
));
}
public function execute(PhutilArgumentParser $args) {
$console = PhutilConsole::getConsole();
$engine_id = $args->getArg('engine');
if (!$engine_id) {
throw new PhutilArgumentUsageException(
"Specify an engine to migrate to with `--engine`. ".
"Use `files engines` to get a list of engines.");
}
$engine = PhabricatorFile::buildEngine($engine_id);
if ($args->getArg('all')) {
if ($args->getArg('names')) {
throw new PhutilArgumentUsageException(
"Specify either a list of files or `--all`, but not both.");
}
$iterator = new LiskMigrationIterator(new PhabricatorFile());
} else if ($args->getArg('names')) {
$iterator = array();
foreach ($args->getArg('names') as $name) {
$name = trim($name);
$id = preg_replace('/^F/i', '', $name);
if (ctype_digit($id)) {
$file = id(new PhabricatorFile())->loadOneWhere(
'id = %d',
$id);
if (!$file) {
throw new PhutilArgumentUsageException(
"No file exists with id '{$name}'.");
}
} else {
$file = id(new PhabricatorFile())->loadOneWhere(
'phid = %d',
$name);
if (!$file) {
throw new PhutilArgumentUsageException(
"No file exists with PHID '{$name}'.");
}
}
$iterator[] = $file;
}
} else {
throw new PhutilArgumentUsageException(
"Either specify a list of files to migrate, or use `--all` ".
"to migrate all files.");
}
$is_dry_run = $args->getArg('dry-run');
$failed = array();
foreach ($iterator as $file) {
$fid = 'F'.$file->getID();
if ($file->getStorageEngine() == $engine_id) {
$console->writeOut(
"%s: Already stored on '%s'\n",
$fid,
$engine_id);
continue;
}
if ($is_dry_run) {
$console->writeOut(
"%s: Would migrate from '%s' to '%s' (dry run)\n",
$fid,
$file->getStorageEngine(),
$engine_id);
continue;
}
$console->writeOut(
"%s: Migrating from '%s' to '%s'...",
$fid,
$file->getStorageEngine(),
$engine_id);
try {
$file->migrateToEngine($engine);
$console->writeOut("done.\n");
} catch (Exception $ex) {
$console->writeOut("failed!\n");
$console->writeErr("%s\n", (string)$ex);
$failed[] = $file;
}
}
if ($failed) {
$console->writeOut("**Failures!**\n");
$ids = array();
foreach ($failed as $file) {
$ids[] = 'F'.$file->getID();
}
$console->writeOut("%s\n", implode(', ', $ids));
return 1;
} else {
$console->writeOut("**Success!**\n");
return 0;
}
}
}

View file

@ -0,0 +1,26 @@
<?php
/*
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
abstract class PhabricatorFilesManagementWorkflow
extends PhutilArgumentWorkflow {
public function isExecutable() {
return true;
}
}

View file

@ -149,44 +149,31 @@ final class PhabricatorFile extends PhabricatorFileDAO {
throw new Exception("No valid storage engines are available!");
}
$file = new PhabricatorFile();
$data_handle = null;
$engine_identifier = null;
$exceptions = array();
foreach ($engines as $engine) {
$engine_class = get_class($engine);
try {
// Perform the actual write.
$data_handle = $engine->writeFile($data, $params);
if (!$data_handle || strlen($data_handle) > 255) {
// This indicates an improperly implemented storage engine.
throw new PhabricatorFileStorageConfigurationException(
"Storage engine '{$engine_class}' executed writeFile() but did ".
"not return a valid handle ('{$data_handle}') to the data: it ".
"must be nonempty and no longer than 255 characters.");
}
$engine_identifier = $engine->getEngineIdentifier();
if (!$engine_identifier || strlen($engine_identifier) > 32) {
throw new PhabricatorFileStorageConfigurationException(
"Storage engine '{$engine_class}' returned an improper engine ".
"identifier '{$engine_identifier}': it must be nonempty ".
"and no longer than 32 characters.");
}
list($engine_identifier, $data_handle) = $file->writeToEngine(
$engine,
$data,
$params);
// We stored the file somewhere so stop trying to write it to other
// places.
break;
} catch (PhabricatorFileStorageConfigurationException $ex) {
// If an engine is outright misconfigured (or misimplemented), raise
// that immediately since it probably needs attention.
throw $ex;
} catch (Exception $ex) {
// If an engine doesn't work, keep trying all the other valid engines
// in case something else works.
phlog($ex);
// If an engine doesn't work, keep trying all the other valid engines
// in case something else works.
$exceptions[$engine_class] = $ex;
}
}
@ -204,7 +191,6 @@ final class PhabricatorFile extends PhabricatorFileDAO {
// (always the case with newFromFileDownload()), store a ''
$authorPHID = idx($params, 'authorPHID');
$file = new PhabricatorFile();
$file->setName($file_name);
$file->setByteSize(strlen($data));
$file->setAuthorPHID($authorPHID);
@ -230,6 +216,63 @@ final class PhabricatorFile extends PhabricatorFileDAO {
return $file;
}
public function migrateToEngine(PhabricatorFileStorageEngine $engine) {
if (!$this->getID() || !$this->getStorageHandle()) {
throw new Exception(
"You can not migrate a file which hasn't yet been saved.");
}
$data = $this->loadFileData();
$params = array(
'name' => $this->getName(),
);
list($new_identifier, $new_handle) = $this->writeToEngine(
$engine,
$data,
$params);
$old_engine = $this->instantiateStorageEngine();
$old_handle = $this->getStorageHandle();
$this->setStorageEngine($new_identifier);
$this->setStorageHandle($new_handle);
$this->save();
$old_engine->deleteFile($old_handle);
return $this;
}
private function writeToEngine(
PhabricatorFileStorageEngine $engine,
$data,
array $params) {
$engine_class = get_class($engine);
$data_handle = $engine->writeFile($data, $params);
if (!$data_handle || strlen($data_handle) > 255) {
// This indicates an improperly implemented storage engine.
throw new PhabricatorFileStorageConfigurationException(
"Storage engine '{$engine_class}' executed writeFile() but did ".
"not return a valid handle ('{$data_handle}') to the data: it ".
"must be nonempty and no longer than 255 characters.");
}
$engine_identifier = $engine->getEngineIdentifier();
if (!$engine_identifier || strlen($engine_identifier) > 32) {
throw new PhabricatorFileStorageConfigurationException(
"Storage engine '{$engine_class}' returned an improper engine ".
"identifier '{$engine_identifier}': it must be nonempty ".
"and no longer than 32 characters.");
}
return array($engine_identifier, $data_handle);
}
public static function newFromFileDownload($uri, $name) {
$uri = new PhutilURI($uri);
@ -402,19 +445,34 @@ final class PhabricatorFile extends PhabricatorFileDAO {
}
protected function instantiateStorageEngine() {
$engines = id(new PhutilSymbolLoader())
->setType('class')
->setAncestorClass('PhabricatorFileStorageEngine')
->selectAndLoadSymbols();
return self::buildEngine($this->getStorageEngine());
}
foreach ($engines as $engine_class) {
$engine = newv($engine_class['name'], array());
if ($engine->getEngineIdentifier() == $this->getStorageEngine()) {
public static function buildEngine($engine_identifier) {
$engines = self::buildAllEngines();
foreach ($engines as $engine) {
if ($engine->getEngineIdentifier() == $engine_identifier) {
return $engine;
}
}
throw new Exception("File's storage engine could be located!");
throw new Exception(
"Storage engine '{$engine_identifier}' could not be located!");
}
public static function buildAllEngines() {
$engines = id(new PhutilSymbolLoader())
->setType('class')
->setConcreteOnly(true)
->setAncestorClass('PhabricatorFileStorageEngine')
->selectAndLoadSymbols();
$results = array();
foreach ($engines as $engine_class) {
$results[] = newv($engine_class['name'], array());
}
return $results;
}
public function getViewableMimeType() {

View file

@ -80,6 +80,22 @@ Technical Documentation}.
You can test that things are correctly configured by going to the Files
application (##/file/##) and uploading files.
= Migrating Files Beteeen Engines =
If you want to move files between storage engines, you can use the `bin/files`
script to perform migrations. For example, suppose you previously used MySQL but
recently set up S3 and want to migrate all your files there. First, migrate one
file to make sure things work:
phabricator/ $ ./bin/files migrate --engine amazon-s3 F12345
If that works properly, you can then migrate everything:
phabricator/ $ ./bin/files migrate --engine amazon-s3 --all
You can use `--dry-run` to show which migrations would be performed without
taking any action. Run `bin/files help` for more options and information.
= Next Steps =
Continue by: