From 918db8c11e58cc2ba5c583025175e4671676d050 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sat, 27 Jan 2018 06:12:03 -0800 Subject: [PATCH] (stable) Add `bin/garbage compact-edges` to compact edges into the new format Summary: Depends on D18947. Ref T13051. This goes through transaction tables and compacts the edge storage into the slim format. I put this on `bin/garbage` instead of `bin/storage` because `bin/storage` has a lot of weird stuff about how it manages databases so that it can run before configuration (e.g., all the `--user`, `--password` type flags for configuring DB connections). Test Plan: Loaded an object with a bunch of transactions. Ran migration. Spot checked table for sanity. Loaded another copy of the object in the web UI, compared the two pages, saw no user-visible changes. Here's a concrete example of the migration effect -- old row: ``` *************************** 44. row *************************** id: 757 phid: PHID-XACT-PSTE-5gnaaway2vnyen5 authorPHID: PHID-USER-cvfydnwadpdj7vdon36z objectPHID: PHID-PSTE-5uj6oqv4kmhtr6ctwcq7 viewPolicy: public editPolicy: PHID-USER-cvfydnwadpdj7vdon36z commentPHID: NULL commentVersion: 0 transactionType: core:edge oldValue: {"PHID-PROJ-wh32nih7q5scvc5lvipv":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-wh32nih7q5scvc5lvipv","dateCreated":"1449170691","seq":"0","dataID":null,"data":[]},"PHID-PROJ-5r2ed5v27xrgltvou5or":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-5r2ed5v27xrgltvou5or","dateCreated":"1449170683","seq":"0","dataID":null,"data":[]},"PHID-PROJ-zfp44q7loir643b5i4v4":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-zfp44q7loir643b5i4v4","dateCreated":"1449170668","seq":"0","dataID":null,"data":[]},"PHID-PROJ-okljqs7prifhajtvia3t":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-okljqs7prifhajtvia3t","dateCreated":"1448902756","seq":"0","dataID":null,"data":[]},"PHID-PROJ-3cuwfuuh4pwqyuof2hhr":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-3cuwfuuh4pwqyuof2hhr","dateCreated":"1448899367","seq":"0","dataID":null,"data":[]},"PHID-PROJ-amvkc5zw2gsy7tyvocug":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-amvkc5zw2gsy7tyvocug","dateCreated":"1448833330","seq":"0","dataID":null,"data":[]}} newValue: {"PHID-PROJ-wh32nih7q5scvc5lvipv":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-wh32nih7q5scvc5lvipv","dateCreated":"1449170691","seq":"0","dataID":null,"data":[]},"PHID-PROJ-5r2ed5v27xrgltvou5or":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-5r2ed5v27xrgltvou5or","dateCreated":"1449170683","seq":"0","dataID":null,"data":[]},"PHID-PROJ-zfp44q7loir643b5i4v4":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-zfp44q7loir643b5i4v4","dateCreated":"1449170668","seq":"0","dataID":null,"data":[]},"PHID-PROJ-okljqs7prifhajtvia3t":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-okljqs7prifhajtvia3t","dateCreated":"1448902756","seq":"0","dataID":null,"data":[]},"PHID-PROJ-3cuwfuuh4pwqyuof2hhr":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-3cuwfuuh4pwqyuof2hhr","dateCreated":"1448899367","seq":"0","dataID":null,"data":[]},"PHID-PROJ-amvkc5zw2gsy7tyvocug":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-amvkc5zw2gsy7tyvocug","dateCreated":"1448833330","seq":"0","dataID":null,"data":[]},"PHID-PROJ-tbowhnwinujwhb346q36":{"dst":"PHID-PROJ-tbowhnwinujwhb346q36","type":41,"data":[]},"PHID-PROJ-izrto7uflimduo6uw2tp":{"dst":"PHID-PROJ-izrto7uflimduo6uw2tp","type":41,"data":[]}} contentSource: {"source":"web","params":[]} metadata: {"edge:type":41} dateCreated: 1450197571 dateModified: 1450197571 ``` New row: ``` *************************** 44. row *************************** id: 757 phid: PHID-XACT-PSTE-5gnaaway2vnyen5 authorPHID: PHID-USER-cvfydnwadpdj7vdon36z objectPHID: PHID-PSTE-5uj6oqv4kmhtr6ctwcq7 viewPolicy: public editPolicy: PHID-USER-cvfydnwadpdj7vdon36z commentPHID: NULL commentVersion: 0 transactionType: core:edge oldValue: [] newValue: ["PHID-PROJ-tbowhnwinujwhb346q36","PHID-PROJ-izrto7uflimduo6uw2tp"] contentSource: {"source":"web","params":[]} metadata: {"edge:type":41} dateCreated: 1450197571 dateModified: 1450197571 ``` Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T13051 Differential Revision: https://secure.phabricator.com/D18948 --- src/__phutil_library_map__.php | 2 + ...ollectorManagementCompactEdgesWorkflow.php | 103 ++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCompactEdgesWorkflow.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index c40f79b052..eaaa4d9e66 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3017,6 +3017,7 @@ phutil_register_library_map(array( 'PhabricatorGDSetupCheck' => 'applications/config/check/PhabricatorGDSetupCheck.php', 'PhabricatorGarbageCollector' => 'infrastructure/daemon/garbagecollector/PhabricatorGarbageCollector.php', 'PhabricatorGarbageCollectorManagementCollectWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCollectWorkflow.php', + 'PhabricatorGarbageCollectorManagementCompactEdgesWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCompactEdgesWorkflow.php', 'PhabricatorGarbageCollectorManagementSetPolicyWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementSetPolicyWorkflow.php', 'PhabricatorGarbageCollectorManagementWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementWorkflow.php', 'PhabricatorGeneralCachePurger' => 'applications/cache/purger/PhabricatorGeneralCachePurger.php', @@ -8484,6 +8485,7 @@ phutil_register_library_map(array( 'PhabricatorGDSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorGarbageCollector' => 'Phobject', 'PhabricatorGarbageCollectorManagementCollectWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow', + 'PhabricatorGarbageCollectorManagementCompactEdgesWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow', 'PhabricatorGarbageCollectorManagementSetPolicyWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow', 'PhabricatorGarbageCollectorManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorGeneralCachePurger' => 'PhabricatorCachePurger', diff --git a/src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCompactEdgesWorkflow.php b/src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCompactEdgesWorkflow.php new file mode 100644 index 0000000000..c47d159850 --- /dev/null +++ b/src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCompactEdgesWorkflow.php @@ -0,0 +1,103 @@ +setName('compact-edges') + ->setExamples('**compact-edges**') + ->setSynopsis( + pht( + 'Rebuild old edge transactions storage to use a more compact '. + 'format.')) + ->setArguments(array()); + } + + public function execute(PhutilArgumentParser $args) { + $tables = id(new PhutilClassMapQuery()) + ->setAncestorClass('PhabricatorApplicationTransaction') + ->execute(); + + foreach ($tables as $table) { + $this->compactEdges($table); + } + + return 0; + } + + private function compactEdges(PhabricatorApplicationTransaction $table) { + $conn = $table->establishConnection('w'); + $class = get_class($table); + + echo tsprintf( + "%s\n", + pht( + 'Rebuilding transactions for "%s"...', + $class)); + + $cursor = 0; + $updated = 0; + while (true) { + $rows = $table->loadAllWhere( + 'transactionType = %s + AND id > %d + AND (oldValue LIKE %> OR newValue LIKE %>) + ORDER BY id ASC LIMIT 100', + PhabricatorTransactions::TYPE_EDGE, + $cursor, + // We're looking for transactions with JSON objects in their value + // fields: the new style transactions have JSON lists instead and + // start with "[" rather than "{". + '{', + '{'); + + if (!$rows) { + break; + } + + foreach ($rows as $row) { + $id = $row->getID(); + + $old = $row->getOldValue(); + $new = $row->getNewValue(); + + if (!is_array($old) || !is_array($new)) { + echo tsprintf( + "%s\n", + pht( + 'Transaction %s (of type %s) has unexpected data, skipping.', + $id, + $class)); + } + + $record = PhabricatorEdgeChangeRecord::newFromTransaction($row); + + $old_data = $record->getModernOldEdgeTransactionData(); + $old_json = phutil_json_encode($old_data); + + $new_data = $record->getModernNewEdgeTransactionData(); + $new_json = phutil_json_encode($new_data); + + queryfx( + $conn, + 'UPDATE %T SET oldValue = %s, newValue = %s WHERE id = %d', + $table->getTableName(), + $old_json, + $new_json, + $id); + + $updated++; + + $cursor = $row->getID(); + } + } + + echo tsprintf( + "%s\n", + pht( + 'Done, compacted %s edge transactions.', + new PhutilNumber($updated))); + } + +}