revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-10 00:42:41 +01:00
Code Issues Releases Wiki Activity

Apply storage patches patch-by-patch, not database-by-database

Browse source 
Summary:
Ref T11044. Sometimes we have a sequence of patches like this:

  - `01.newtable.sql`: Adds a new table to Files.
  - `02.movedata.php`: Moves some data that used to live in Tokens to the new table.

This is fairly rare, but not unheard of. More commonly, we can have this sequence:

  - `03.newtable.sql`: Add a new column to Phame.
  - `04.setvalue.php`: Copy data into that new column.

In both cases, when applying database-by-database, we can get into trouble.

  - In the first case, if Files is on a different master, we'll try to move data into a new table before creating the table.
  - In the second case, if Phame is on a different master, the PHP patch will connect to it before we add the new column.

In either case, we try to interact with tables or columns which don't exist yet.

Instead, apply each patch in order, to all databases which need it. So we'll apply `01.newtable.sql` EVERYWHERE first, then move on.

In the case of PHP patches, we also now only apply them once, since they never make schema changes. It should normally be OK to apply them more than once safely, but this is a little faster and a little safer if we ever make a mistake.

Test Plan:
  - Ran `bin/storage upgrade` on single-host and clustered setups.
  - Initialized new storage on single-host and clustered setups.
  - Upgraded again after initialization.
  - Ran with `--apply`.
  - Ran with `--dry-run`.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T11044

Differential Revision: https://secure.phabricator.com/D16912
This commit is contained in:
epriestley 2016-11-22 05:37:50 -08:00
parent e6bfa1bd23
commit f89f708692
3 changed files with 204 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/infrastructure/storage/management/PhabricatorStoragePatch.php
View file

@ -48,4 +48,8 @@ final class PhabricatorStoragePatch extends Phobject {
return $this->dead; return $this->dead;
} }
public function getIsGlobalPatch() {
return ($this->getType() == 'php');
}
} }

14
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php
View file

@ -75,14 +75,14 @@ final class PhabricatorStorageManagementUpgradeWorkflow
$apis = $this->getMasterAPIs(); $apis = $this->getMasterAPIs();
foreach ($apis as $api) { $this->upgradeSchemata($apis, $apply_only, $no_quickstart, $init_only);
$this->upgradeSchemata($api, $apply_only, $no_quickstart, $init_only);
if ($no_adjust || $init_only || $apply_only) { if ($no_adjust || $init_only || $apply_only) {
$console->writeOut( $console->writeOut(
"%s\n", "%s\n",
pht('Declining to apply storage adjustments.')); pht('Declining to apply storage adjustments.'));
} else { } else {
foreach ($apis as $api) {
$err = $this->adjustSchemata($api, false); $err = $this->adjustSchemata($api, false);
if ($err) { if ($err) {
return $err; return $err;

270
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php
View file

@ -819,58 +819,79 @@ abstract class PhabricatorStorageManagementWorkflow
} }
final protected function upgradeSchemata( final protected function upgradeSchemata(
PhabricatorStorageManagementAPI $api, array $apis,
$apply_only = null, $apply_only = null,
$no_quickstart = false, $no_quickstart = false,
$init_only = false) { $init_only = false) {
$lock = $this->lock($api); $locks = array();
foreach ($apis as $api) {
$ref_key = $api->getRef()->getRefKey();
$locks[] = $this->lock($api);
}
try { try {
$this->doUpgradeSchemata($api, $apply_only, $no_quickstart, $init_only); $this->doUpgradeSchemata($apis, $apply_only, $no_quickstart, $init_only);
} catch (Exception $ex) { } catch (Exception $ex) {
$lock->unlock(); foreach ($locks as $lock) {
$lock->unlock();
}
throw $ex; throw $ex;
} }
$lock->unlock(); foreach ($locks as $lock) {
$lock->unlock();
}
} }
final private function doUpgradeSchemata( final private function doUpgradeSchemata(
PhabricatorStorageManagementAPI $api, array $apis,
$apply_only, $apply_only,
$no_quickstart, $no_quickstart,
$init_only) { $init_only) {
$patches = $this->patches; $patches = $this->patches;
$is_dryrun = $this->dryRun;
$applied = $api->getAppliedPatches(); $api_map = array();
if ($applied === null) { foreach ($apis as $api) {
if ($this->dryRun) { $api_map[$api->getRef()->getRefKey()] = $api;
echo pht( }
"DRYRUN: Patch metadata storage doesn't exist yet, ".
"it would be created.\n"); foreach ($api_map as $ref_key => $api) {
return 0; $applied = $api->getAppliedPatches();
$needs_init = ($applied === null);
if (!$needs_init) {
continue;
}
if ($is_dryrun) {
echo tsprintf(
"%s\n",
pht(
'DRYRUN: Storage on host "%s" does not exist yet, so it '.
'would be created.',
$ref_key));
continue;
} }
if ($apply_only) { if ($apply_only) {
throw new PhutilArgumentUsageException( throw new PhutilArgumentUsageException(
pht( pht(
'Storage has not been initialized yet, you must initialize '. 'Storage on host "%s" has not been initialized yet. You must '.
'storage before selectively applying patches.')); 'initialize storage before selectively applying patches.',
return 1; $ref_key));
} }
// If we're initializing storage for the first time, track it so that // If we're initializing storage for the first time on any host, track
// we can give the user a nicer experience during the subsequent // it so that we can give the user a nicer experience during the
// adjustment phase. // subsequent adjustment phase.
$this->didInitialize = true; $this->didInitialize = true;
$legacy = $api->getLegacyPatches($patches); $legacy = $api->getLegacyPatches($patches);
if ($legacy || $no_quickstart || $init_only) { if ($legacy || $no_quickstart || $init_only) {
// If we have legacy patches, we can't quickstart. // If we have legacy patches, we can't quickstart.
$api->createDatabase('meta_data'); $api->createDatabase('meta_data');
$api->createTable( $api->createTable(
'meta_data', 'meta_data',
@ -884,7 +905,12 @@ abstract class PhabricatorStorageManagementWorkflow
$api->markPatchApplied($patch); $api->markPatchApplied($patch);
} }
} else { } else {
echo pht('Loading quickstart template...')."\n"; echo tsprintf(
"%s\n",
pht(
'Loading quickstart template onto "%s"...',
$ref_key));
$root = dirname(phutil_get_library_root('phabricator')); $root = dirname(phutil_get_library_root('phabricator'));
$sql = $root.'/resources/sql/quickstart.sql'; $sql = $root.'/resources/sql/quickstart.sql';
$api->applyPatchSQL($sql); $api->applyPatchSQL($sql);
@ -896,95 +922,180 @@ abstract class PhabricatorStorageManagementWorkflow
return 0; return 0;
} }
$applied = $api->getAppliedPatches(); $applied_map = array();
$applied = array_fuse($applied); foreach ($api_map as $ref_key => $api) {
$applied = $api->getAppliedPatches();
$skip_mark = false; // If we still have nothing applied, this is a dry run and we didn't
if ($apply_only) { // actually initialize storage. Here, just do nothing.
if (isset($applied[$apply_only])) { if ($applied === null) {
if ($is_dryrun) {
unset($applied[$apply_only]); continue;
$skip_mark = true; } else {
throw new Exception(
if (!$this->force && !$this->dryRun) {
echo phutil_console_wrap(
pht( pht(
"Patch '%s' has already been applied. Are you sure you want ". 'Database initialization on host "%s" applied no patches!',
"to apply it again? This may put your storage in a state ". $ref_key));
"that the upgrade scripts can not automatically manage.",
$apply_only));
if (!phutil_console_confirm(pht('Apply patch again?'))) {
echo pht('Cancelled.')."\n";
return 1;
}
} }
} }
$applied = array_fuse($applied);
if ($apply_only) {
if (isset($applied[$apply_only])) {
if (!$this->force && !$is_dryrun) {
echo phutil_console_wrap(
pht(
'Patch "%s" has already been applied on host "%s". Are you '.
'sure you want to apply it again? This may put your storage '.
'in a state that the upgrade scripts can not automatically '.
'manage.',
$apply_only,
$ref_key));
if (!phutil_console_confirm(pht('Apply patch again?'))) {
echo pht('Cancelled.')."\n";
return 1;
}
}
// Mark this patch as not yet applied on this host.
unset($applied[$apply_only]);
}
}
$applied_map[$ref_key] = $applied;
} }
// If we're applying only a specific patch, select just that patch.
if ($apply_only) {
$patches = array_select_keys($patches, array($apply_only));
}
// Apply each patch to each database. We apply patches patch-by-patch,
// not database-by-database: for each patch we apply it to every database,
// then move to the next patch.
// We must do this because ".php" patches may depend on ".sql" patches
// being up to date on all masters, and that will work fine if we put each
// patch on every host before moving on. If we try to bring database hosts
// up to date one at a time we can end up in a big mess.
$duration_map = array();
while (true) { while (true) {
$applied_something = false; $applied_something = false;
foreach ($patches as $key => $patch) { foreach ($patches as $key => $patch) {
if (isset($applied[$key])) { // First, check if any databases need this patch. We can just skip it
// if it has already been applied everywhere.
$need_patch = array();
foreach ($applied_map as $ref_key => $applied) {
if (isset($applied[$key])) {
continue;
}
$need_patch[] = $ref_key;
}
if (!$need_patch) {
unset($patches[$key]); unset($patches[$key]);
continue; continue;
} }
if ($apply_only && $apply_only != $key) { // Check if we can apply this patch yet. Before we can apply a patch,
unset($patches[$key]); // all of the dependencies for the patch must have been applied on all
continue; // databases. Requiring that all databases stay in sync prevents one
} // database from racing ahead if it happens to get a patch that nothing
// else has yet.
$can_apply = true; $missing_patch = null;
foreach ($patch->getAfter() as $after) { foreach ($patch->getAfter() as $after) {
if (empty($applied[$after])) { foreach ($applied_map as $ref_key => $applied) {
if ($apply_only) { if (isset($applied[$after])) {
echo pht( // This database already has the patch. We can apply it to
"Unable to apply patch '%s' because it depends ". // other databases but don't need to apply it here.
"on patch '%s', which has not been applied.\n", continue;
$apply_only,
$after);
return 1;
} }
$can_apply = false;
break; $missing_patch = $after;
break 2;
} }
} }
if (!$can_apply) { if ($missing_patch) {
continue; if ($apply_only) {
echo tsprintf(
"%s\n",
pht(
'Unable to apply patch "%s" because it depends on patch '.
'"%s", which has not been applied on some hosts: %s.',
$apply_only,
$missing_patch,
implode(', ', $need_patch)));
return 1;
} else {
// Some databases are missing the dependencies, so keep trying
// other patches instead. If everything goes right, we'll apply the
// dependencies and then come back and apply this patch later.
continue;
}
} }
$applied_something = true; $is_global = $patch->getIsGlobalPatch();
$patch_apis = array_select_keys($api_map, $need_patch);
foreach ($patch_apis as $ref_key => $api) {
if ($is_global) {
// If this is a global patch which we previously applied, just
// read the duration from the map without actually applying
// the patch.
$duration = idx($duration_map, $key);
} else {
$duration = null;
}
if ($this->dryRun) { if ($duration === null) {
echo pht("DRYRUN: Would apply patch '%s'.", $key)."\n"; if ($is_dryrun) {
} else { echo tsprintf(
echo pht("Applying patch '%s'...", $key)."\n"; "%s\n",
pht(
'DRYRUN: Would apply patch "%s" to host "%s".',
$key,
$ref_key));
} else {
echo tsprintf(
"%s\n",
pht(
'Applying patch "%s" to host "%s"...',
$key,
$ref_key));
}
$t_begin = microtime(true); $t_begin = microtime(true);
$api->applyPatch($patch); $api->applyPatch($patch);
$t_end = microtime(true); $t_end = microtime(true);
if (!$skip_mark) { $duration = ($t_end - $t_begin);
$duration_map[$key] = $duration;
}
// If we're explicitly reapplying this patch, we don't need to
// mark it as applied.
if (!isset($applied_map[$ref_key][$key])) {
$api->markPatchApplied($key, ($t_end - $t_begin)); $api->markPatchApplied($key, ($t_end - $t_begin));
$applied_map[$ref_key][$key] = true;
} }
} }
// We applied this everywhere, so we're done with the patch.
unset($patches[$key]); unset($patches[$key]);
$applied[$key] = true; $applied_something = true;
} }
if (!$applied_something) { if (!$applied_something) {
if (count($patches)) { if ($patches) {
throw new Exception( throw new Exception(
pht( pht(
'Some patches could not be applied to "%s": %s', 'Some patches could not be applied: %s',
$api->getRef()->getRefKey(),
implode(', ', array_keys($patches)))); implode(', ', array_keys($patches))));
} else if (!$this->dryRun && !$apply_only) { } else if (!$is_dryrun && !$apply_only) {
echo pht( echo pht(
'Storage is up to date on "%s". Use "%s" for details.', 'Storage is up to date. Use "%s" for details.',
$api->getRef()->getRefKey(),
'storage status')."\n"; 'storage status')."\n";
} }
break; break;
@ -1013,7 +1124,12 @@ abstract class PhabricatorStorageManagementWorkflow
* @return PhabricatorGlobalLock * @return PhabricatorGlobalLock
*/ */
final protected function lock(PhabricatorStorageManagementAPI $api) { final protected function lock(PhabricatorStorageManagementAPI $api) {
return PhabricatorGlobalLock::newLock(__CLASS__) // Although we're holding this lock on different databases so it could
// have the same name on each as far as the database is concerned, the
// locks would be the same within this process.
$lock_name = 'adjust/'.$api->getRef()->getRefKey();
return PhabricatorGlobalLock::newLock($lock_name)
->useSpecificConnection($api->getConn(null)) ->useSpecificConnection($api->getConn(null))
->lock(); ->lock();
} }