revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-25 16:22:43 +01:00
Code Issues Releases Wiki Activity

Add a bin/files purge workflow

Browse source 
Summary:
We can lose file data through various means; one reasonable way is if files get deleted from disk with 'local-disk' storage. If data goes missing,

Ref T3265. Also, reduce some code duplication.

Test Plan:
Ran `bin/files purge`, `bin/files migrate`, `bin/files rebuild` with various args.

Deleted a file with "local-disk" storage, ran `bin/files purge`, made sure it got picked up.

Reviewers: dctrwatson, btrahan

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T3265

Differential Revision: https://secure.phabricator.com/D6068
This commit is contained in:
epriestley 2013-05-29 06:28:57 -07:00
parent b91c863780
commit 0f070236bd
7 changed files with 131 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
scripts/files/manage_files.php
View file

@ -17,8 +17,9 @@ $args->parseStandardArguments();
$workflows = array(
new PhabricatorFilesManagementEnginesWorkflow(),
new PhabricatorFilesManagementMigrateWorkflow(),
new PhutilHelpArgumentWorkflow(),
new PhabricatorFilesManagementRebuildWorkflow(),
new PhabricatorFilesManagementPurgeWorkflow(),
new PhutilHelpArgumentWorkflow(),
);
$args->parseWorkflows($workflows);

2
src/__phutil_library_map__.php
View file

@ -1014,6 +1014,7 @@ phutil_register_library_map(array(
'PhabricatorFilesConfigOptions' => 'applications/files/config/PhabricatorFilesConfigOptions.php',
'PhabricatorFilesManagementEnginesWorkflow' => 'applications/files/management/PhabricatorFilesManagementEnginesWorkflow.php',
'PhabricatorFilesManagementMigrateWorkflow' => 'applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php',
'PhabricatorFilesManagementPurgeWorkflow' => 'applications/files/management/PhabricatorFilesManagementPurgeWorkflow.php',
'PhabricatorFilesManagementRebuildWorkflow' => 'applications/files/management/PhabricatorFilesManagementRebuildWorkflow.php',
'PhabricatorFilesManagementWorkflow' => 'applications/files/management/PhabricatorFilesManagementWorkflow.php',
'PhabricatorFlag' => 'applications/flag/storage/PhabricatorFlag.php',
@ -2814,6 +2815,7 @@ phutil_register_library_map(array(
'PhabricatorFilesConfigOptions' => 'PhabricatorApplicationConfigOptions',
'PhabricatorFilesManagementEnginesWorkflow' => 'PhabricatorFilesManagementWorkflow',
'PhabricatorFilesManagementMigrateWorkflow' => 'PhabricatorFilesManagementWorkflow',
'PhabricatorFilesManagementPurgeWorkflow' => 'PhabricatorFilesManagementWorkflow',
'PhabricatorFilesManagementRebuildWorkflow' => 'PhabricatorFilesManagementWorkflow',
'PhabricatorFilesManagementWorkflow' => 'PhutilArgumentWorkflow',
'PhabricatorFlag' => 'PhabricatorFlagDAO',

35
src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php
View file

@ -41,39 +41,8 @@ final class PhabricatorFilesManagementMigrateWorkflow
$engine = PhabricatorFile::buildEngine($engine_id);
if ($args->getArg('all')) {
if ($args->getArg('names')) {
throw new PhutilArgumentUsageException(
"Specify either a list of files or `--all`, but not both.");
}
$iterator = new LiskMigrationIterator(new PhabricatorFile());
} else if ($args->getArg('names')) {
$iterator = array();
foreach ($args->getArg('names') as $name) {
$name = trim($name);
$id = preg_replace('/^F/i', '', $name);
if (ctype_digit($id)) {
$file = id(new PhabricatorFile())->loadOneWhere(
'id = %d',
$id);
if (!$file) {
throw new PhutilArgumentUsageException(
"No file exists with id '{$name}'.");
}
} else {
$file = id(new PhabricatorFile())->loadOneWhere(
'phid = %d',
$name);
if (!$file) {
throw new PhutilArgumentUsageException(
"No file exists with PHID '{$name}'.");
}
}
$iterator[] = $file;
}
} else {
$iterator = $this->buildIterator($args);
if (!$iterator) {
throw new PhutilArgumentUsageException(
"Either specify a list of files to migrate, or use `--all` ".
"to migrate all files.");

69
src/applications/files/management/PhabricatorFilesManagementPurgeWorkflow.php Normal file
View file

@ -0,0 +1,69 @@
<?php
final class PhabricatorFilesManagementPurgeWorkflow
extends PhabricatorFilesManagementWorkflow {
public function didConstruct() {
$this
->setName('purge')
->setSynopsis('Delete files with missing data.')
->setArguments(
array(
array(
'name' => 'all',
'help' => 'Update all files.',
),
array(
'name' => 'dry-run',
'help' => 'Show what would be updated.',
),
array(
'name' => 'names',
'wildcard' => true,
),
));
}
public function execute(PhutilArgumentParser $args) {
$console = PhutilConsole::getConsole();
$iterator = $this->buildIterator($args);
if (!$iterator) {
throw new PhutilArgumentUsageException(
"Either specify a list of files to purge, or use `--all` ".
"to purge all files.");
}
$is_dry_run = $args->getArg('dry-run');
foreach ($iterator as $file) {
$fid = 'F'.$file->getID();
try {
$file->loadFileData();
$okay = true;
} catch (Exception $ex) {
$okay = false;
}
if ($okay) {
$console->writeOut(
"%s: File data is OK, not purging.\n",
$fid);
} else {
if ($is_dry_run) {
$console->writeOut(
"%s: Would purge (dry run).\n",
$fid);
} else {
$console->writeOut(
"%s: Purging.\n",
$fid);
$file->delete();
}
}
}
return 0;
}
}

37
src/applications/files/management/PhabricatorFilesManagementRebuildWorkflow.php
View file

@ -13,12 +13,6 @@ final class PhabricatorFilesManagementRebuildWorkflow
'name' => 'all',
'help' => 'Update all files.',
),
array(
'name' => 'id',
'wildcard' => true,
'help' => 'Update the given file. You can specify this flag '.
'multiple times.',
),
array(
'name' => 'dry-run',
'help' => 'Show what would be updated.',
@ -31,37 +25,18 @@ final class PhabricatorFilesManagementRebuildWorkflow
'name' => 'rebuild-dimensions',
'help' => 'Rebuild image dimension information.',
),
array(
'name' => 'names',
'wildcard' => true,
),
));
}
public function execute(PhutilArgumentParser $args) {
$console = PhutilConsole::getConsole();
if ($args->getArg('all')) {
if ($args->getArg('id')) {
throw new PhutilArgumentUsageException(
"Specify either a list of files or `--all`, but not both.");
}
$iterator = new LiskMigrationIterator(new PhabricatorFile());
} else if ($args->getArg('id')) {
$iterator = array();
foreach ($args->getArg('id') as $id) {
$id = trim($id);
$id = preg_replace('/^F/i', '', $id);
if (ctype_digit($id)) {
$file = id(new PhabricatorFile())->loadOneWhere(
'id = %d',
$id);
if (!$file) {
throw new PhutilArgumentUsageException(
"No file exists with ID '{$id}'.");
}
}
$iterator[] = $file;
}
} else {
$iterator = $this->buildIterator($args);
if (!$iterator) {
throw new PhutilArgumentUsageException(
"Either specify a list of files to update, or use `--all` ".
"to update all files.");

43
src/applications/files/management/PhabricatorFilesManagementWorkflow.php
View file

@ -7,4 +7,47 @@ abstract class PhabricatorFilesManagementWorkflow
return true;
}
protected function buildIterator(PhutilArgumentParser $args) {
if ($args->getArg('all')) {
if ($args->getArg('names')) {
throw new PhutilArgumentUsageException(
"Specify either a list of files or `--all`, but not both.");
}
return new LiskMigrationIterator(new PhabricatorFile());
}
if ($args->getArg('names')) {
$iterator = array();
foreach ($args->getArg('names') as $name) {
$name = trim($name);
$id = preg_replace('/^F/i', '', $name);
if (ctype_digit($id)) {
$file = id(new PhabricatorFile())->loadOneWhere(
'id = %d',
$id);
if (!$file) {
throw new PhutilArgumentUsageException(
"No file exists with ID '{$name}'.");
}
} else {
$file = id(new PhabricatorFile())->loadOneWhere(
'phid = %d',
$name);
if (!$file) {
throw new PhutilArgumentUsageException(
"No file exists with PHID '{$name}'.");
}
}
$iterator[] = $file;
}
return $iterator;
}
return null;
}
}

6
src/applications/files/storage/PhabricatorFile.php
View file

@ -429,7 +429,13 @@ final class PhabricatorFile extends PhabricatorFileDAO
// If this is the only file using the storage, delete storage
if (!$other_file) {
$engine = $this->instantiateStorageEngine();
try {
$engine->deleteFile($this->getStorageHandle());
} catch (Exception $ex) {
// In the worst case, we're leaving some data stranded in a storage
// engine, which is fine.
phlog($ex);
}
}
return $ret;