mirror of
https://we.phorge.it/source/phorge.git
synced 2024-11-22 14:52:41 +01:00
Add bin/garbage compact-edges
to compact edges into the new format
Summary: Depends on D18947. Ref T13051. This goes through transaction tables and compacts the edge storage into the slim format. I put this on `bin/garbage` instead of `bin/storage` because `bin/storage` has a lot of weird stuff about how it manages databases so that it can run before configuration (e.g., all the `--user`, `--password` type flags for configuring DB connections). Test Plan: Loaded an object with a bunch of transactions. Ran migration. Spot checked table for sanity. Loaded another copy of the object in the web UI, compared the two pages, saw no user-visible changes. Here's a concrete example of the migration effect -- old row: ``` *************************** 44. row *************************** id: 757 phid: PHID-XACT-PSTE-5gnaaway2vnyen5 authorPHID: PHID-USER-cvfydnwadpdj7vdon36z objectPHID: PHID-PSTE-5uj6oqv4kmhtr6ctwcq7 viewPolicy: public editPolicy: PHID-USER-cvfydnwadpdj7vdon36z commentPHID: NULL commentVersion: 0 transactionType: core:edge oldValue: {"PHID-PROJ-wh32nih7q5scvc5lvipv":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-wh32nih7q5scvc5lvipv","dateCreated":"1449170691","seq":"0","dataID":null,"data":[]},"PHID-PROJ-5r2ed5v27xrgltvou5or":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-5r2ed5v27xrgltvou5or","dateCreated":"1449170683","seq":"0","dataID":null,"data":[]},"PHID-PROJ-zfp44q7loir643b5i4v4":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-zfp44q7loir643b5i4v4","dateCreated":"1449170668","seq":"0","dataID":null,"data":[]},"PHID-PROJ-okljqs7prifhajtvia3t":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-okljqs7prifhajtvia3t","dateCreated":"1448902756","seq":"0","dataID":null,"data":[]},"PHID-PROJ-3cuwfuuh4pwqyuof2hhr":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-3cuwfuuh4pwqyuof2hhr","dateCreated":"1448899367","seq":"0","dataID":null,"data":[]},"PHID-PROJ-amvkc5zw2gsy7tyvocug":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-amvkc5zw2gsy7tyvocug","dateCreated":"1448833330","seq":"0","dataID":null,"data":[]}} newValue: {"PHID-PROJ-wh32nih7q5scvc5lvipv":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-wh32nih7q5scvc5lvipv","dateCreated":"1449170691","seq":"0","dataID":null,"data":[]},"PHID-PROJ-5r2ed5v27xrgltvou5or":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-5r2ed5v27xrgltvou5or","dateCreated":"1449170683","seq":"0","dataID":null,"data":[]},"PHID-PROJ-zfp44q7loir643b5i4v4":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-zfp44q7loir643b5i4v4","dateCreated":"1449170668","seq":"0","dataID":null,"data":[]},"PHID-PROJ-okljqs7prifhajtvia3t":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-okljqs7prifhajtvia3t","dateCreated":"1448902756","seq":"0","dataID":null,"data":[]},"PHID-PROJ-3cuwfuuh4pwqyuof2hhr":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-3cuwfuuh4pwqyuof2hhr","dateCreated":"1448899367","seq":"0","dataID":null,"data":[]},"PHID-PROJ-amvkc5zw2gsy7tyvocug":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-amvkc5zw2gsy7tyvocug","dateCreated":"1448833330","seq":"0","dataID":null,"data":[]},"PHID-PROJ-tbowhnwinujwhb346q36":{"dst":"PHID-PROJ-tbowhnwinujwhb346q36","type":41,"data":[]},"PHID-PROJ-izrto7uflimduo6uw2tp":{"dst":"PHID-PROJ-izrto7uflimduo6uw2tp","type":41,"data":[]}} contentSource: {"source":"web","params":[]} metadata: {"edge:type":41} dateCreated: 1450197571 dateModified: 1450197571 ``` New row: ``` *************************** 44. row *************************** id: 757 phid: PHID-XACT-PSTE-5gnaaway2vnyen5 authorPHID: PHID-USER-cvfydnwadpdj7vdon36z objectPHID: PHID-PSTE-5uj6oqv4kmhtr6ctwcq7 viewPolicy: public editPolicy: PHID-USER-cvfydnwadpdj7vdon36z commentPHID: NULL commentVersion: 0 transactionType: core:edge oldValue: [] newValue: ["PHID-PROJ-tbowhnwinujwhb346q36","PHID-PROJ-izrto7uflimduo6uw2tp"] contentSource: {"source":"web","params":[]} metadata: {"edge:type":41} dateCreated: 1450197571 dateModified: 1450197571 ``` Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13051 Differential Revision: https://secure.phabricator.com/D18948
This commit is contained in:
parent
e5639a8ed9
commit
6d2d1d3a97
2 changed files with 105 additions and 0 deletions
|
@ -3017,6 +3017,7 @@ phutil_register_library_map(array(
|
||||||
'PhabricatorGDSetupCheck' => 'applications/config/check/PhabricatorGDSetupCheck.php',
|
'PhabricatorGDSetupCheck' => 'applications/config/check/PhabricatorGDSetupCheck.php',
|
||||||
'PhabricatorGarbageCollector' => 'infrastructure/daemon/garbagecollector/PhabricatorGarbageCollector.php',
|
'PhabricatorGarbageCollector' => 'infrastructure/daemon/garbagecollector/PhabricatorGarbageCollector.php',
|
||||||
'PhabricatorGarbageCollectorManagementCollectWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCollectWorkflow.php',
|
'PhabricatorGarbageCollectorManagementCollectWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCollectWorkflow.php',
|
||||||
|
'PhabricatorGarbageCollectorManagementCompactEdgesWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCompactEdgesWorkflow.php',
|
||||||
'PhabricatorGarbageCollectorManagementSetPolicyWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementSetPolicyWorkflow.php',
|
'PhabricatorGarbageCollectorManagementSetPolicyWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementSetPolicyWorkflow.php',
|
||||||
'PhabricatorGarbageCollectorManagementWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementWorkflow.php',
|
'PhabricatorGarbageCollectorManagementWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementWorkflow.php',
|
||||||
'PhabricatorGeneralCachePurger' => 'applications/cache/purger/PhabricatorGeneralCachePurger.php',
|
'PhabricatorGeneralCachePurger' => 'applications/cache/purger/PhabricatorGeneralCachePurger.php',
|
||||||
|
@ -8484,6 +8485,7 @@ phutil_register_library_map(array(
|
||||||
'PhabricatorGDSetupCheck' => 'PhabricatorSetupCheck',
|
'PhabricatorGDSetupCheck' => 'PhabricatorSetupCheck',
|
||||||
'PhabricatorGarbageCollector' => 'Phobject',
|
'PhabricatorGarbageCollector' => 'Phobject',
|
||||||
'PhabricatorGarbageCollectorManagementCollectWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow',
|
'PhabricatorGarbageCollectorManagementCollectWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow',
|
||||||
|
'PhabricatorGarbageCollectorManagementCompactEdgesWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow',
|
||||||
'PhabricatorGarbageCollectorManagementSetPolicyWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow',
|
'PhabricatorGarbageCollectorManagementSetPolicyWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow',
|
||||||
'PhabricatorGarbageCollectorManagementWorkflow' => 'PhabricatorManagementWorkflow',
|
'PhabricatorGarbageCollectorManagementWorkflow' => 'PhabricatorManagementWorkflow',
|
||||||
'PhabricatorGeneralCachePurger' => 'PhabricatorCachePurger',
|
'PhabricatorGeneralCachePurger' => 'PhabricatorCachePurger',
|
||||||
|
|
|
@ -0,0 +1,103 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
final class PhabricatorGarbageCollectorManagementCompactEdgesWorkflow
|
||||||
|
extends PhabricatorGarbageCollectorManagementWorkflow {
|
||||||
|
|
||||||
|
protected function didConstruct() {
|
||||||
|
$this
|
||||||
|
->setName('compact-edges')
|
||||||
|
->setExamples('**compact-edges**')
|
||||||
|
->setSynopsis(
|
||||||
|
pht(
|
||||||
|
'Rebuild old edge transactions storage to use a more compact '.
|
||||||
|
'format.'))
|
||||||
|
->setArguments(array());
|
||||||
|
}
|
||||||
|
|
||||||
|
public function execute(PhutilArgumentParser $args) {
|
||||||
|
$tables = id(new PhutilClassMapQuery())
|
||||||
|
->setAncestorClass('PhabricatorApplicationTransaction')
|
||||||
|
->execute();
|
||||||
|
|
||||||
|
foreach ($tables as $table) {
|
||||||
|
$this->compactEdges($table);
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function compactEdges(PhabricatorApplicationTransaction $table) {
|
||||||
|
$conn = $table->establishConnection('w');
|
||||||
|
$class = get_class($table);
|
||||||
|
|
||||||
|
echo tsprintf(
|
||||||
|
"%s\n",
|
||||||
|
pht(
|
||||||
|
'Rebuilding transactions for "%s"...',
|
||||||
|
$class));
|
||||||
|
|
||||||
|
$cursor = 0;
|
||||||
|
$updated = 0;
|
||||||
|
while (true) {
|
||||||
|
$rows = $table->loadAllWhere(
|
||||||
|
'transactionType = %s
|
||||||
|
AND id > %d
|
||||||
|
AND (oldValue LIKE %> OR newValue LIKE %>)
|
||||||
|
ORDER BY id ASC LIMIT 100',
|
||||||
|
PhabricatorTransactions::TYPE_EDGE,
|
||||||
|
$cursor,
|
||||||
|
// We're looking for transactions with JSON objects in their value
|
||||||
|
// fields: the new style transactions have JSON lists instead and
|
||||||
|
// start with "[" rather than "{".
|
||||||
|
'{',
|
||||||
|
'{');
|
||||||
|
|
||||||
|
if (!$rows) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach ($rows as $row) {
|
||||||
|
$id = $row->getID();
|
||||||
|
|
||||||
|
$old = $row->getOldValue();
|
||||||
|
$new = $row->getNewValue();
|
||||||
|
|
||||||
|
if (!is_array($old) || !is_array($new)) {
|
||||||
|
echo tsprintf(
|
||||||
|
"%s\n",
|
||||||
|
pht(
|
||||||
|
'Transaction %s (of type %s) has unexpected data, skipping.',
|
||||||
|
$id,
|
||||||
|
$class));
|
||||||
|
}
|
||||||
|
|
||||||
|
$record = PhabricatorEdgeChangeRecord::newFromTransaction($row);
|
||||||
|
|
||||||
|
$old_data = $record->getModernOldEdgeTransactionData();
|
||||||
|
$old_json = phutil_json_encode($old_data);
|
||||||
|
|
||||||
|
$new_data = $record->getModernNewEdgeTransactionData();
|
||||||
|
$new_json = phutil_json_encode($new_data);
|
||||||
|
|
||||||
|
queryfx(
|
||||||
|
$conn,
|
||||||
|
'UPDATE %T SET oldValue = %s, newValue = %s WHERE id = %d',
|
||||||
|
$table->getTableName(),
|
||||||
|
$old_json,
|
||||||
|
$new_json,
|
||||||
|
$id);
|
||||||
|
|
||||||
|
$updated++;
|
||||||
|
|
||||||
|
$cursor = $row->getID();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
echo tsprintf(
|
||||||
|
"%s\n",
|
||||||
|
pht(
|
||||||
|
'Done, compacted %s edge transactions.',
|
||||||
|
new PhutilNumber($updated)));
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
Loading…
Reference in a new issue