2011-01-24 20:01:53 +01:00
|
|
|
<?php
|
|
|
|
|
Support storage of Differential hunk data in Files
Summary:
Ref T12932. For long-lived installs, one of the largest tables tends to be the hunk data table. Although it doesn't grow tremendously fast, it's also well suited to storage in Files instead of the database (infrequent access, relatively large blobs of data, mostly one-at-a-time access), and earlier work anticipated eventually adding support for Files storage.
Make Files storage work, and provide `bin/differential migrate-hunk` to manually test/migrate hunks. This is currently the only way hunks get moved to file storage, but I expect to add a GC step which moves them to File storage after 30 days shortly.
The immediate motivation for this is to relieve storage pressure on db001/db002 so we have more headroom for deploying the Ferret engine and its larger indexes (see also T12819).
Test Plan:
- Used `bin/differential migrate-hunk` to move a hunk to and from file storage, verified it survived intact.
- Downloaded the actual stored file, sanity-checked it. Verified permissions.
- Destroyed a diff with `bin/remove destroy`, saw the hunk and file storage destroyed.
- Verified that going from file -> text destroys the old file properly with `migrate-hunk --trace ...`.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T12932
Differential Revision: https://secure.phabricator.com/D18584
2017-09-11 17:08:19 +02:00
|
|
|
abstract class DifferentialHunk
|
|
|
|
extends DifferentialDAO
|
|
|
|
implements
|
|
|
|
PhabricatorPolicyInterface,
|
|
|
|
PhabricatorDestructibleInterface {
|
2011-01-24 20:01:53 +01:00
|
|
|
|
|
|
|
protected $changesetID;
|
|
|
|
protected $oldOffset;
|
|
|
|
protected $oldLen;
|
|
|
|
protected $newOffset;
|
|
|
|
protected $newLen;
|
|
|
|
|
2014-04-14 21:06:26 +02:00
|
|
|
private $changeset;
|
2015-05-07 01:43:32 +02:00
|
|
|
private $splitLines;
|
2015-03-24 21:11:37 +01:00
|
|
|
private $structuredLines;
|
|
|
|
private $structuredFiles = array();
|
2014-04-14 21:06:26 +02:00
|
|
|
|
2014-04-14 21:06:20 +02:00
|
|
|
const FLAG_LINES_ADDED = 1;
|
|
|
|
const FLAG_LINES_REMOVED = 2;
|
|
|
|
const FLAG_LINES_STABLE = 4;
|
|
|
|
|
2012-04-28 08:00:30 +02:00
|
|
|
public function getAddedLines() {
|
2012-06-27 19:44:29 +02:00
|
|
|
return $this->makeContent($include = '+');
|
2012-04-28 08:00:30 +02:00
|
|
|
}
|
|
|
|
|
2013-10-04 15:37:32 +02:00
|
|
|
public function getRemovedLines() {
|
|
|
|
return $this->makeContent($include = '-');
|
|
|
|
}
|
|
|
|
|
2011-01-24 20:01:53 +01:00
|
|
|
public function makeNewFile() {
|
2012-06-27 19:44:29 +02:00
|
|
|
return implode('', $this->makeContent($include = ' +'));
|
2011-01-24 20:01:53 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
public function makeOldFile() {
|
2012-06-27 19:44:29 +02:00
|
|
|
return implode('', $this->makeContent($include = ' -'));
|
2011-01-24 20:01:53 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
public function makeChanges() {
|
2012-06-27 19:44:29 +02:00
|
|
|
return implode('', $this->makeContent($include = '-+'));
|
2011-01-24 20:01:53 +01:00
|
|
|
}
|
|
|
|
|
2015-03-24 21:11:37 +01:00
|
|
|
public function getStructuredOldFile() {
|
|
|
|
return $this->getStructuredFile('-');
|
|
|
|
}
|
|
|
|
|
|
|
|
public function getStructuredNewFile() {
|
|
|
|
return $this->getStructuredFile('+');
|
|
|
|
}
|
|
|
|
|
|
|
|
private function getStructuredFile($kind) {
|
|
|
|
if ($kind !== '+' && $kind !== '-') {
|
|
|
|
throw new Exception(
|
|
|
|
pht(
|
|
|
|
'Structured file kind should be "+" or "-", got "%s".',
|
|
|
|
$kind));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!isset($this->structuredFiles[$kind])) {
|
|
|
|
if ($kind == '+') {
|
|
|
|
$number = $this->newOffset;
|
|
|
|
} else {
|
|
|
|
$number = $this->oldOffset;
|
|
|
|
}
|
|
|
|
|
|
|
|
$lines = $this->getStructuredLines();
|
|
|
|
|
|
|
|
// NOTE: We keep the "\ No newline at end of file" line if it appears
|
|
|
|
// after a line which is not excluded. For example, if we're constructing
|
|
|
|
// the "+" side of the diff, we want to ignore this one since it's
|
|
|
|
// relevant only to the "-" side of the diff:
|
|
|
|
//
|
|
|
|
// - x
|
|
|
|
// \ No newline at end of file
|
|
|
|
// + x
|
|
|
|
//
|
|
|
|
// ...but we want to keep this one:
|
|
|
|
//
|
|
|
|
// - x
|
|
|
|
// + x
|
|
|
|
// \ No newline at end of file
|
|
|
|
|
|
|
|
$file = array();
|
|
|
|
$keep = true;
|
|
|
|
foreach ($lines as $line) {
|
|
|
|
switch ($line['type']) {
|
|
|
|
case ' ':
|
|
|
|
case $kind:
|
|
|
|
$file[$number++] = $line;
|
|
|
|
$keep = true;
|
|
|
|
break;
|
|
|
|
case '\\':
|
|
|
|
if ($keep) {
|
|
|
|
// Strip the actual newline off the line's text.
|
|
|
|
$text = $file[$number - 1]['text'];
|
|
|
|
$text = rtrim($text, "\r\n");
|
|
|
|
$file[$number - 1]['text'] = $text;
|
|
|
|
|
|
|
|
$file[$number++] = $line;
|
|
|
|
$keep = false;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
$keep = false;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
$this->structuredFiles[$kind] = $file;
|
|
|
|
}
|
|
|
|
|
|
|
|
return $this->structuredFiles[$kind];
|
|
|
|
}
|
|
|
|
|
2015-05-07 01:43:32 +02:00
|
|
|
public function getSplitLines() {
|
|
|
|
if ($this->splitLines === null) {
|
|
|
|
$this->splitLines = phutil_split_lines($this->getChanges());
|
|
|
|
}
|
|
|
|
return $this->splitLines;
|
|
|
|
}
|
|
|
|
|
2015-05-07 23:09:41 +02:00
|
|
|
public function getStructuredLines() {
|
2015-03-24 21:11:37 +01:00
|
|
|
if ($this->structuredLines === null) {
|
2015-05-07 01:43:32 +02:00
|
|
|
$lines = $this->getSplitLines();
|
2015-03-24 21:11:37 +01:00
|
|
|
|
|
|
|
$structured = array();
|
|
|
|
foreach ($lines as $line) {
|
|
|
|
if (empty($line[0])) {
|
|
|
|
// TODO: Can we just get rid of this?
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
$structured[] = array(
|
|
|
|
'type' => $line[0],
|
|
|
|
'text' => substr($line, 1),
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
$this->structuredLines = $structured;
|
|
|
|
}
|
|
|
|
|
|
|
|
return $this->structuredLines;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2014-04-14 21:06:20 +02:00
|
|
|
public function getContentWithMask($mask) {
|
|
|
|
$include = array();
|
|
|
|
|
|
|
|
if (($mask & self::FLAG_LINES_ADDED)) {
|
|
|
|
$include[] = '+';
|
|
|
|
}
|
|
|
|
|
|
|
|
if (($mask & self::FLAG_LINES_REMOVED)) {
|
|
|
|
$include[] = '-';
|
|
|
|
}
|
|
|
|
|
|
|
|
if (($mask & self::FLAG_LINES_STABLE)) {
|
|
|
|
$include[] = ' ';
|
|
|
|
}
|
|
|
|
|
|
|
|
$include = implode('', $include);
|
|
|
|
|
|
|
|
return implode('', $this->makeContent($include));
|
|
|
|
}
|
|
|
|
|
2012-06-27 19:44:29 +02:00
|
|
|
final private function makeContent($include) {
|
2015-05-07 01:43:32 +02:00
|
|
|
$lines = $this->getSplitLines();
|
2011-01-24 20:01:53 +01:00
|
|
|
$results = array();
|
2012-03-22 22:13:48 +01:00
|
|
|
|
2015-05-07 01:43:32 +02:00
|
|
|
$include_map = array();
|
|
|
|
for ($ii = 0; $ii < strlen($include); $ii++) {
|
|
|
|
$include_map[$include[$ii]] = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (isset($include_map['+'])) {
|
|
|
|
$n = $this->newOffset;
|
|
|
|
} else {
|
|
|
|
$n = $this->oldOffset;
|
|
|
|
}
|
2012-03-22 22:13:48 +01:00
|
|
|
|
|
|
|
$use_next_newline = false;
|
2011-01-24 20:01:53 +01:00
|
|
|
foreach ($lines as $line) {
|
2012-06-27 19:44:29 +02:00
|
|
|
if (!isset($line[0])) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ($line[0] == '\\') {
|
|
|
|
if ($use_next_newline) {
|
|
|
|
$results[last_key($results)] = rtrim(end($results), "\n");
|
2012-03-22 22:13:48 +01:00
|
|
|
}
|
2015-05-07 01:43:32 +02:00
|
|
|
} else if (empty($include_map[$line[0]])) {
|
2012-06-27 19:44:29 +02:00
|
|
|
$use_next_newline = false;
|
|
|
|
} else {
|
|
|
|
$use_next_newline = true;
|
2015-05-07 01:43:32 +02:00
|
|
|
$results[$n] = substr($line, 1);
|
2011-01-24 20:01:53 +01:00
|
|
|
}
|
2012-03-22 22:13:48 +01:00
|
|
|
|
2015-05-07 01:43:32 +02:00
|
|
|
if ($line[0] == ' ' || isset($include_map[$line[0]])) {
|
2012-06-27 19:44:29 +02:00
|
|
|
$n++;
|
|
|
|
}
|
2012-03-22 22:13:48 +01:00
|
|
|
}
|
|
|
|
|
2012-06-27 19:44:29 +02:00
|
|
|
return $results;
|
2011-01-24 20:01:53 +01:00
|
|
|
}
|
|
|
|
|
2014-04-14 21:06:26 +02:00
|
|
|
public function getChangeset() {
|
|
|
|
return $this->assertAttached($this->changeset);
|
|
|
|
}
|
|
|
|
|
|
|
|
public function attachChangeset(DifferentialChangeset $changeset) {
|
|
|
|
$this->changeset = $changeset;
|
|
|
|
return $this;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/* -( PhabricatorPolicyInterface )----------------------------------------- */
|
|
|
|
|
|
|
|
|
|
|
|
public function getCapabilities() {
|
|
|
|
return array(
|
|
|
|
PhabricatorPolicyCapability::CAN_VIEW,
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
public function getPolicy($capability) {
|
|
|
|
return $this->getChangeset()->getPolicy($capability);
|
|
|
|
}
|
|
|
|
|
|
|
|
public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
|
|
|
|
return $this->getChangeset()->hasAutomaticCapability($capability, $viewer);
|
|
|
|
}
|
|
|
|
|
Support storage of Differential hunk data in Files
Summary:
Ref T12932. For long-lived installs, one of the largest tables tends to be the hunk data table. Although it doesn't grow tremendously fast, it's also well suited to storage in Files instead of the database (infrequent access, relatively large blobs of data, mostly one-at-a-time access), and earlier work anticipated eventually adding support for Files storage.
Make Files storage work, and provide `bin/differential migrate-hunk` to manually test/migrate hunks. This is currently the only way hunks get moved to file storage, but I expect to add a GC step which moves them to File storage after 30 days shortly.
The immediate motivation for this is to relieve storage pressure on db001/db002 so we have more headroom for deploying the Ferret engine and its larger indexes (see also T12819).
Test Plan:
- Used `bin/differential migrate-hunk` to move a hunk to and from file storage, verified it survived intact.
- Downloaded the actual stored file, sanity-checked it. Verified permissions.
- Destroyed a diff with `bin/remove destroy`, saw the hunk and file storage destroyed.
- Verified that going from file -> text destroys the old file properly with `migrate-hunk --trace ...`.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T12932
Differential Revision: https://secure.phabricator.com/D18584
2017-09-11 17:08:19 +02:00
|
|
|
|
|
|
|
/* -( PhabricatorDestructibleInterface )----------------------------------- */
|
|
|
|
|
|
|
|
|
|
|
|
public function destroyObjectPermanently(
|
|
|
|
PhabricatorDestructionEngine $engine) {
|
|
|
|
$this->delete();
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2011-01-24 20:01:53 +01:00
|
|
|
}
|