Add a script to migrate files between storage engines

Summary: Quora requested this (moving to S3) but it's also clearly a good idea.

Test Plan:
Ran with various valid/invalid options to test options. Error/sanity checking seemed OK.

Migrated individual local files.

Migrated all my local files back and forth between engines several times.

Uploaded some new files.

Reviewers: btrahan, vrana

Reviewed By: vrana

CC: aran

Maniphest Tasks: T1950

Differential Revision: https://secure.phabricator.com/D3808

bin/files

@@ -0,0 +1 @@
+../scripts/files/manage_files.php

scripts/files/manage_files.php

@@ -0,0 +1,39 @@
+#!/usr/bin/env php
+<?php
+
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$root = dirname(dirname(dirname(__FILE__)));
+require_once $root.'/scripts/__init_script__.php';
+
+$args = new PhutilArgumentParser($argv);
+$args->setTagline('manage files');
+$args->setSynopsis(<<<EOSYNOPSIS
+**files** __command__ [__options__]
+    Manage Phabricator file storage.
+
+EOSYNOPSIS
+  );
+$args->parseStandardArguments();
+
+$workflows = array(
+  new PhabricatorFilesManagementEnginesWorkflow(),
+  new PhabricatorFilesManagementMigrateWorkflow(),
+  new PhutilHelpArgumentWorkflow(),
+);
+
+$args->parseWorkflows($workflows);

src/__phutil_library_map__.php

@@ -755,6 +755,9 @@ phutil_register_library_map(array(
     'PhabricatorFileUploadController' => 'applications/files/controller/PhabricatorFileUploadController.php',
     'PhabricatorFileUploadException' => 'applications/files/exception/PhabricatorFileUploadException.php',
     'PhabricatorFileUploadView' => 'applications/files/view/PhabricatorFileUploadView.php',
+    'PhabricatorFilesManagementEnginesWorkflow' => 'applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php',
+    'PhabricatorFilesManagementMigrateWorkflow' => 'applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php',
+    'PhabricatorFilesManagementWorkflow' => 'applications/files/management/PhabricatorFilesManagementWorkflow.php',
     'PhabricatorFlag' => 'applications/flag/storage/PhabricatorFlag.php',
     'PhabricatorFlagColor' => 'applications/flag/constants/PhabricatorFlagColor.php',
     'PhabricatorFlagConstants' => 'applications/flag/constants/PhabricatorFlagConstants.php',
@@ -1949,6 +1952,9 @@ phutil_register_library_map(array(
     'PhabricatorFileUploadController' => 'PhabricatorFileController',
     'PhabricatorFileUploadException' => 'Exception',
     'PhabricatorFileUploadView' => 'AphrontView',
+    'PhabricatorFilesManagementEnginesWorkflow' => 'PhabricatorFilesManagementWorkflow',
+    'PhabricatorFilesManagementMigrateWorkflow' => 'PhabricatorFilesManagementWorkflow',
+    'PhabricatorFilesManagementWorkflow' => 'PhutilArgumentWorkflow',
     'PhabricatorFlag' => 'PhabricatorFlagDAO',
     'PhabricatorFlagColor' => 'PhabricatorFlagConstants',
     'PhabricatorFlagController' => 'PhabricatorController',

src/applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php

@@ -0,0 +1,46 @@
+<?php
+
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+final class PhabricatorFilesManagementEnginesWorkflow
+  extends PhabricatorFilesManagementWorkflow {
+
+  public function didConstruct() {
+    $this
+      ->setName('engines')
+      ->setSynopsis('List available storage engines.')
+      ->setArguments(array());
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $console = PhutilConsole::getConsole();
+
+    $engines = PhabricatorFile::buildAllEngines();
+    if (!$engines) {
+      throw new Exception("No storage engines are available.");
+    }
+
+    foreach ($engines as $engine) {
+      $console->writeOut(
+        "%s\n",
+        $engine->getEngineIdentifier());
+    }
+
+    return 0;
+  }
+
+}

src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php

@@ -0,0 +1,153 @@
+<?php
+
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+final class PhabricatorFilesManagementMigrateWorkflow
+  extends PhabricatorFilesManagementWorkflow {
+
+  public function didConstruct() {
+    $this
+      ->setName('migrate')
+      ->setSynopsis('Migrate files between storage engines.')
+      ->setArguments(
+        array(
+          array(
+            'name'      => 'engine',
+            'param'     => 'storage_engine',
+            'help'      => 'Migrate to the named storage engine.',
+          ),
+          array(
+            'name'      => 'dry-run',
+            'help'      => 'Show what would be migrated.',
+          ),
+          array(
+            'name'      => 'all',
+            'help'      => 'Migrate all files.',
+          ),
+          array(
+            'name'      => 'names',
+            'wildcard'  => true,
+          ),
+        ));
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $console = PhutilConsole::getConsole();
+
+    $engine_id = $args->getArg('engine');
+    if (!$engine_id) {
+      throw new PhutilArgumentUsageException(
+        "Specify an engine to migrate to with `--engine`. ".
+        "Use `files engines` to get a list of engines.");
+    }
+
+    $engine = PhabricatorFile::buildEngine($engine_id);
+
+    if ($args->getArg('all')) {
+      if ($args->getArg('names')) {
+        throw new PhutilArgumentUsageException(
+          "Specify either a list of files or `--all`, but not both.");
+      }
+      $iterator = new LiskMigrationIterator(new PhabricatorFile());
+    } else if ($args->getArg('names')) {
+      $iterator = array();
+
+      foreach ($args->getArg('names') as $name) {
+        $name = trim($name);
+
+        $id = preg_replace('/^F/i', '', $name);
+        if (ctype_digit($id)) {
+          $file = id(new PhabricatorFile())->loadOneWhere(
+            'id = %d',
+            $id);
+          if (!$file) {
+            throw new PhutilArgumentUsageException(
+              "No file exists with id '{$name}'.");
+          }
+        } else {
+          $file = id(new PhabricatorFile())->loadOneWhere(
+            'phid = %d',
+            $name);
+          if (!$file) {
+            throw new PhutilArgumentUsageException(
+              "No file exists with PHID '{$name}'.");
+          }
+        }
+        $iterator[] = $file;
+      }
+    } else {
+      throw new PhutilArgumentUsageException(
+        "Either specify a list of files to migrate, or use `--all` ".
+        "to migrate all files.");
+    }
+
+    $is_dry_run = $args->getArg('dry-run');
+
+    $failed = array();
+
+    foreach ($iterator as $file) {
+      $fid = 'F'.$file->getID();
+
+      if ($file->getStorageEngine() == $engine_id) {
+        $console->writeOut(
+          "%s: Already stored on '%s'\n",
+          $fid,
+          $engine_id);
+        continue;
+      }
+
+      if ($is_dry_run) {
+        $console->writeOut(
+          "%s: Would migrate from '%s' to '%s' (dry run)\n",
+          $fid,
+          $file->getStorageEngine(),
+          $engine_id);
+        continue;
+      }
+
+      $console->writeOut(
+        "%s: Migrating from '%s' to '%s'...",
+        $fid,
+        $file->getStorageEngine(),
+        $engine_id);
+
+      try {
+        $file->migrateToEngine($engine);
+        $console->writeOut("done.\n");
+      } catch (Exception $ex) {
+        $console->writeOut("failed!\n");
+        $console->writeErr("%s\n", (string)$ex);
+        $failed[] = $file;
+      }
+    }
+
+    if ($failed) {
+      $console->writeOut("**Failures!**\n");
+      $ids = array();
+      foreach ($failed as $file) {
+        $ids[] = 'F'.$file->getID();
+      }
+      $console->writeOut("%s\n", implode(', ', $ids));
+
+      return 1;
+    } else {
+      $console->writeOut("**Success!**\n");
+      return 0;
+    }
+  }
+
+}

src/applications/files/management/PhabricatorFilesManagementWorkflow.php

@@ -0,0 +1,26 @@
+<?php
+
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+abstract class PhabricatorFilesManagementWorkflow
+  extends PhutilArgumentWorkflow {
+
+  public function isExecutable() {
+    return true;
+  }
+
+}

src/applications/files/storage/PhabricatorFile.php

@@ -149,44 +149,31 @@ final class PhabricatorFile extends PhabricatorFileDAO {
       throw new Exception("No valid storage engines are available!");
     }
 
+    $file = new PhabricatorFile();
+
     $data_handle = null;
     $engine_identifier = null;
     $exceptions = array();
     foreach ($engines as $engine) {
       $engine_class = get_class($engine);
       try {
-        // Perform the actual write.
+        list($engine_identifier, $data_handle) = $file->writeToEngine(
-        $data_handle = $engine->writeFile($data, $params);
+          $engine,
-        if (!$data_handle || strlen($data_handle) > 255) {
+          $data,
-          // This indicates an improperly implemented storage engine.
+          $params);
-          throw new PhabricatorFileStorageConfigurationException(
-            "Storage engine '{$engine_class}' executed writeFile() but did ".
-            "not return a valid handle ('{$data_handle}') to the data: it ".
-            "must be nonempty and no longer than 255 characters.");
-        }
-
-        $engine_identifier = $engine->getEngineIdentifier();
-        if (!$engine_identifier || strlen($engine_identifier) > 32) {
-          throw new PhabricatorFileStorageConfigurationException(
-            "Storage engine '{$engine_class}' returned an improper engine ".
-            "identifier '{$engine_identifier}': it must be nonempty ".
-            "and no longer than 32 characters.");
-        }
 
         // We stored the file somewhere so stop trying to write it to other
         // places.
         break;
-
       } catch (PhabricatorFileStorageConfigurationException $ex) {
         // If an engine is outright misconfigured (or misimplemented), raise
         // that immediately since it probably needs attention.
         throw $ex;
-
       } catch (Exception $ex) {
-        // If an engine doesn't work, keep trying all the other valid engines
-        // in case something else works.
         phlog($ex);
 
+        // If an engine doesn't work, keep trying all the other valid engines
+        // in case something else works.
         $exceptions[$engine_class] = $ex;
       }
     }
@@ -204,7 +191,6 @@ final class PhabricatorFile extends PhabricatorFileDAO {
     // (always the case with newFromFileDownload()), store a ''
     $authorPHID = idx($params, 'authorPHID');
 
-    $file = new PhabricatorFile();
     $file->setName($file_name);
     $file->setByteSize(strlen($data));
     $file->setAuthorPHID($authorPHID);
@@ -230,6 +216,63 @@ final class PhabricatorFile extends PhabricatorFileDAO {
     return $file;
   }
 
+  public function migrateToEngine(PhabricatorFileStorageEngine $engine) {
+    if (!$this->getID() || !$this->getStorageHandle()) {
+      throw new Exception(
+        "You can not migrate a file which hasn't yet been saved.");
+    }
+
+    $data = $this->loadFileData();
+    $params = array(
+      'name' => $this->getName(),
+    );
+
+    list($new_identifier, $new_handle) = $this->writeToEngine(
+      $engine,
+      $data,
+      $params);
+
+    $old_engine = $this->instantiateStorageEngine();
+    $old_handle = $this->getStorageHandle();
+
+    $this->setStorageEngine($new_identifier);
+    $this->setStorageHandle($new_handle);
+    $this->save();
+
+    $old_engine->deleteFile($old_handle);
+
+    return $this;
+  }
+
+  private function writeToEngine(
+    PhabricatorFileStorageEngine $engine,
+    $data,
+    array $params) {
+
+    $engine_class = get_class($engine);
+
+    $data_handle = $engine->writeFile($data, $params);
+
+    if (!$data_handle || strlen($data_handle) > 255) {
+      // This indicates an improperly implemented storage engine.
+      throw new PhabricatorFileStorageConfigurationException(
+        "Storage engine '{$engine_class}' executed writeFile() but did ".
+        "not return a valid handle ('{$data_handle}') to the data: it ".
+        "must be nonempty and no longer than 255 characters.");
+    }
+
+    $engine_identifier = $engine->getEngineIdentifier();
+    if (!$engine_identifier || strlen($engine_identifier) > 32) {
+      throw new PhabricatorFileStorageConfigurationException(
+        "Storage engine '{$engine_class}' returned an improper engine ".
+        "identifier '{$engine_identifier}': it must be nonempty ".
+        "and no longer than 32 characters.");
+    }
+
+    return array($engine_identifier, $data_handle);
+  }
+
+
   public static function newFromFileDownload($uri, $name) {
     $uri = new PhutilURI($uri);
 
@@ -402,19 +445,34 @@ final class PhabricatorFile extends PhabricatorFileDAO {
   }
 
   protected function instantiateStorageEngine() {
-    $engines = id(new PhutilSymbolLoader())
+    return self::buildEngine($this->getStorageEngine());
-      ->setType('class')
+  }
-      ->setAncestorClass('PhabricatorFileStorageEngine')
-      ->selectAndLoadSymbols();
 
-    foreach ($engines as $engine_class) {
+  public static function buildEngine($engine_identifier) {
-      $engine = newv($engine_class['name'], array());
+    $engines = self::buildAllEngines();
-      if ($engine->getEngineIdentifier() == $this->getStorageEngine()) {
+    foreach ($engines as $engine) {
+      if ($engine->getEngineIdentifier() == $engine_identifier) {
         return $engine;
       }
     }
 
-    throw new Exception("File's storage engine could be located!");
+    throw new Exception(
+      "Storage engine '{$engine_identifier}' could not be located!");
+  }
+
+  public static function buildAllEngines() {
+    $engines = id(new PhutilSymbolLoader())
+      ->setType('class')
+      ->setConcreteOnly(true)
+      ->setAncestorClass('PhabricatorFileStorageEngine')
+      ->selectAndLoadSymbols();
+
+    $results = array();
+    foreach ($engines as $engine_class) {
+      $results[] = newv($engine_class['name'], array());
+    }
+
+    return $results;
   }
 
   public function getViewableMimeType() {

src/docs/configuration/configuring_file_storage.diviner

@@ -80,6 +80,22 @@ Technical Documentation}.
 You can test that things are correctly configured by going to the Files
 application (##/file/##) and uploading files.
 
+= Migrating Files Beteeen Engines =
+
+If you want to move files between storage engines, you can use the `bin/files`
+script to perform migrations. For example, suppose you previously used MySQL but
+recently set up S3 and want to migrate all your files there. First, migrate one
+file to make sure things work:
+
+  phabricator/ $ ./bin/files migrate --engine amazon-s3 F12345
+
+If that works properly, you can then migrate everything:
+
+  phabricator/ $ ./bin/files migrate --engine amazon-s3 --all
+
+You can use `--dry-run` to show which migrations would be performed without
+taking any action. Run `bin/files help` for more options and information.
+
 = Next Steps =
 
 Continue by: