revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2025-01-25 14:08:19 +01:00
Code Issues Releases Wiki Activity

(stable) Promote 2018 Week 20

Browse source
This commit is contained in:
epriestley 2018-05-18 17:23:36 -07:00
commit 9d0adf6563
16 changed files with 665 additions and 258 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/__phutil_library_map__.php
View file

@ -432,6 +432,7 @@ phutil_register_library_map(array(
'DifferentialChangesSinceLastUpdateField' => 'applications/differential/customfield/DifferentialChangesSinceLastUpdateField.php',
'DifferentialChangeset' => 'applications/differential/storage/DifferentialChangeset.php',
'DifferentialChangesetDetailView' => 'applications/differential/view/DifferentialChangesetDetailView.php',
'DifferentialChangesetEngine' => 'applications/differential/engine/DifferentialChangesetEngine.php',
'DifferentialChangesetFileTreeSideNavBuilder' => 'applications/differential/view/DifferentialChangesetFileTreeSideNavBuilder.php',
'DifferentialChangesetHTMLRenderer' => 'applications/differential/render/DifferentialChangesetHTMLRenderer.php',
'DifferentialChangesetListController' => 'applications/differential/controller/DifferentialChangesetListController.php',
@ -2869,6 +2870,7 @@ phutil_register_library_map(array(
'PhabricatorDifferentialExtractWorkflow' => 'applications/differential/management/PhabricatorDifferentialExtractWorkflow.php',
'PhabricatorDifferentialManagementWorkflow' => 'applications/differential/management/PhabricatorDifferentialManagementWorkflow.php',
'PhabricatorDifferentialMigrateHunkWorkflow' => 'applications/differential/management/PhabricatorDifferentialMigrateHunkWorkflow.php',
'PhabricatorDifferentialRebuildChangesetsWorkflow' => 'applications/differential/management/PhabricatorDifferentialRebuildChangesetsWorkflow.php',
'PhabricatorDifferentialRevisionTestDataGenerator' => 'applications/differential/lipsum/PhabricatorDifferentialRevisionTestDataGenerator.php',
'PhabricatorDiffusionApplication' => 'applications/diffusion/application/PhabricatorDiffusionApplication.php',
'PhabricatorDiffusionBlameSetting' => 'applications/settings/setting/PhabricatorDiffusionBlameSetting.php',
@ -4032,6 +4034,7 @@ phutil_register_library_map(array(
'PhabricatorPygmentSetupCheck' => 'applications/config/check/PhabricatorPygmentSetupCheck.php',
'PhabricatorQuery' => 'infrastructure/query/PhabricatorQuery.php',
'PhabricatorQueryConstraint' => 'infrastructure/query/constraint/PhabricatorQueryConstraint.php',
'PhabricatorQueryIterator' => 'infrastructure/storage/lisk/PhabricatorQueryIterator.php',
'PhabricatorQueryOrderItem' => 'infrastructure/query/order/PhabricatorQueryOrderItem.php',
'PhabricatorQueryOrderTestCase' => 'infrastructure/query/order/__tests__/PhabricatorQueryOrderTestCase.php',
'PhabricatorQueryOrderVector' => 'infrastructure/query/order/PhabricatorQueryOrderVector.php',
@ -5728,6 +5731,7 @@ phutil_register_library_map(array(
'PhabricatorDestructibleInterface',
),
'DifferentialChangesetDetailView' => 'AphrontView',
'DifferentialChangesetEngine' => 'Phobject',
'DifferentialChangesetFileTreeSideNavBuilder' => 'Phobject',
'DifferentialChangesetHTMLRenderer' => 'DifferentialChangesetRenderer',
'DifferentialChangesetListController' => 'DifferentialController',
@ -8529,6 +8533,7 @@ phutil_register_library_map(array(
'PhabricatorDifferentialExtractWorkflow' => 'PhabricatorDifferentialManagementWorkflow',
'PhabricatorDifferentialManagementWorkflow' => 'PhabricatorManagementWorkflow',
'PhabricatorDifferentialMigrateHunkWorkflow' => 'PhabricatorDifferentialManagementWorkflow',
'PhabricatorDifferentialRebuildChangesetsWorkflow' => 'PhabricatorDifferentialManagementWorkflow',
'PhabricatorDifferentialRevisionTestDataGenerator' => 'PhabricatorTestDataGenerator',
'PhabricatorDiffusionApplication' => 'PhabricatorApplication',
'PhabricatorDiffusionBlameSetting' => 'PhabricatorInternalSetting',
@ -9876,6 +9881,7 @@ phutil_register_library_map(array(
'PhabricatorPygmentSetupCheck' => 'PhabricatorSetupCheck',
'PhabricatorQuery' => 'Phobject',
'PhabricatorQueryConstraint' => 'Phobject',
'PhabricatorQueryIterator' => 'PhutilBufferedIterator',
'PhabricatorQueryOrderItem' => 'Phobject',
'PhabricatorQueryOrderTestCase' => 'PhabricatorTestCase',
'PhabricatorQueryOrderVector' => array(

13
src/applications/dashboard/controller/PhabricatorDashboardProfileController.php
View file

@ -44,13 +44,16 @@ abstract class PhabricatorDashboardProfileController
}
protected function buildApplicationCrumbs() {
$dashboard = $this->getDashboard();
$id = $dashboard->getID();
$dashboard_uri = $this->getApplicationURI("/view/{$id}/");
$crumbs = parent::buildApplicationCrumbs();
$crumbs->addTextCrumb($dashboard->getName(), $dashboard_uri);
$crumbs->setBorder(true);
$dashboard = $this->getDashboard();
if ($dashboard) {
$id = $dashboard->getID();
$dashboard_uri = $this->getApplicationURI("/view/{$id}/");
$crumbs->addTextCrumb($dashboard->getName(), $dashboard_uri);
}
return $crumbs;
}

101
src/applications/differential/controller/DifferentialRevisionViewController.php
View file

@ -5,6 +5,7 @@ final class DifferentialRevisionViewController
private $revisionID;
private $changesetCount;
private $hiddenChangesets;
public function shouldAllowPublic() {
return true;
@ -169,6 +170,7 @@ final class DifferentialRevisionViewController
}
$handles = $this->loadViewerHandles($object_phids);
$warnings = array();
$request_uri = $request->getRequestURI();
@ -203,11 +205,19 @@ final class DifferentialRevisionViewController
pht('Expand All Files'))),
);
$warning = id(new PHUIInfoView())
$warnings[] = id(new PHUIInfoView())
->setTitle(pht('Large Diff'))
->setSeverity(PHUIInfoView::SEVERITY_WARNING)
->appendChild($message);
$folded_changesets = $changesets;
} else {
$folded_changesets = array();
}
// Don't hide or fold changesets which have inline comments.
$hidden_changesets = $this->hiddenChangesets;
if ($hidden_changesets || $folded_changesets) {
$old = array_select_keys($changesets, $old_ids);
$new = array_select_keys($changesets, $new_ids);
@ -222,16 +232,47 @@ final class DifferentialRevisionViewController
$new,
$revision);
$visible_changesets = array();
foreach ($inlines as $inline) {
$changeset_id = $inline->getChangesetID();
if (isset($changesets[$changeset_id])) {
$visible_changesets[$changeset_id] = $changesets[$changeset_id];
if (!isset($changesets[$changeset_id])) {
continue;
}
unset($hidden_changesets[$changeset_id]);
unset($folded_changesets[$changeset_id]);
}
} else {
$warning = null;
$visible_changesets = $changesets;
}
// If we would hide only one changeset, don't hide anything. The notice
// we'd render about it is about the same size as the changeset.
if (count($hidden_changesets) < 2) {
$hidden_changesets = array();
}
// Update the set of hidden changesets, since we may have just un-hidden
// some of them.
if ($hidden_changesets) {
$warnings[] = id(new PHUIInfoView())
->setTitle(pht('Showing Only Differences'))
->setSeverity(PHUIInfoView::SEVERITY_NOTICE)
->appendChild(
pht(
'This revision modifies %s more files that are hidden because '.
'they were not modified between selected diffs and they have no '.
'inline comments.',
phutil_count($hidden_changesets)));
}
// Compute the unfolded changesets. By default, everything is unfolded.
$unfolded_changesets = $changesets;
foreach ($folded_changesets as $changeset_id => $changeset) {
unset($unfolded_changesets[$changeset_id]);
}
// Throw away any hidden changesets.
foreach ($hidden_changesets as $changeset_id => $changeset) {
unset($changesets[$changeset_id]);
unset($unfolded_changesets[$changeset_id]);
}
$commit_hashes = mpull($diffs, 'getSourceControlBaseRevision');
@ -267,7 +308,7 @@ final class DifferentialRevisionViewController
if ($repository) {
$symbol_indexes = $this->buildSymbolIndexes(
$repository,
$visible_changesets);
$unfolded_changesets);
} else {
$symbol_indexes = array();
}
@ -328,7 +369,7 @@ final class DifferentialRevisionViewController
} else {
$changeset_view = id(new DifferentialChangesetListView())
->setChangesets($changesets)
->setVisibleChangesets($visible_changesets)
->setVisibleChangesets($unfolded_changesets)
->setStandaloneURI('/differential/changeset/')
->setRawFileURIs(
'/differential/changeset/?view=old',
@ -405,7 +446,7 @@ final class DifferentialRevisionViewController
$toc_view = $this->buildTableOfContents(
$changesets,
$visible_changesets,
$unfolded_changesets,
$target->loadCoverageMap($viewer));
// Attach changesets to each reviewer so we can show which Owners package
@ -520,7 +561,7 @@ final class DifferentialRevisionViewController
$footer[] = array(
$anchor,
$warning,
$warnings,
$tab_view,
$changeset_view,
);
@ -807,6 +848,7 @@ final class DifferentialRevisionViewController
DifferentialDiff $target,
DifferentialDiff $diff_vs = null,
PhabricatorRepository $repository = null) {
$viewer = $this->getViewer();
$load_diffs = array($target);
if ($diff_vs) {
@ -814,7 +856,7 @@ final class DifferentialRevisionViewController
}
$raw_changesets = id(new DifferentialChangesetQuery())
->setViewer($this->getRequest()->getUser())
->setViewer($viewer)
->withDiffs($load_diffs)
->execute();
$changeset_groups = mgroup($raw_changesets, 'getDiffID');
@ -822,17 +864,19 @@ final class DifferentialRevisionViewController
$changesets = idx($changeset_groups, $target->getID(), array());
$changesets = mpull($changesets, null, 'getID');
$refs = array();
$vs_map = array();
$refs = array();
$vs_map = array();
$vs_changesets = array();
$must_compare = array();
if ($diff_vs) {
$vs_id = $diff_vs->getID();
$vs_id = $diff_vs->getID();
$vs_changesets_path_map = array();
foreach (idx($changeset_groups, $vs_id, array()) as $changeset) {
$path = $changeset->getAbsoluteRepositoryPath($repository, $diff_vs);
$vs_changesets_path_map[$path] = $changeset;
$vs_changesets[$changeset->getID()] = $changeset;
}
foreach ($changesets as $key => $changeset) {
$path = $changeset->getAbsoluteRepositoryPath($repository, $target);
if (isset($vs_changesets_path_map[$path])) {
@ -841,15 +885,20 @@ final class DifferentialRevisionViewController
$refs[$changeset->getID()] =
$changeset->getID().'/'.$vs_changesets_path_map[$path]->getID();
unset($vs_changesets_path_map[$path]);
$must_compare[] = $changeset->getID();
} else {
$refs[$changeset->getID()] = $changeset->getID();
}
}
foreach ($vs_changesets_path_map as $path => $changeset) {
$changesets[$changeset->getID()] = $changeset;
$vs_map[$changeset->getID()] = -1;
$refs[$changeset->getID()] = $changeset->getID().'/-1';
$vs_map[$changeset->getID()] = -1;
$refs[$changeset->getID()] = $changeset->getID().'/-1';
}
} else {
foreach ($changesets as $changeset) {
$refs[$changeset->getID()] = $changeset->getID();
@ -858,13 +907,25 @@ final class DifferentialRevisionViewController
$changesets = msort($changesets, 'getSortKey');
// See T13137. When displaying the diff between two updates, hide any
// changesets which haven't actually changed.
$this->hiddenChangesets = array();
foreach ($must_compare as $changeset_id) {
$changeset = $changesets[$changeset_id];
$vs_changeset = $vs_changesets[$vs_map[$changeset_id]];
if ($changeset->hasSameEffectAs($vs_changeset)) {
$this->hiddenChangesets[$changeset_id] = $changesets[$changeset_id];
}
}
return array($changesets, $vs_map, $vs_changesets, $refs);
}
private function buildSymbolIndexes(
PhabricatorRepository $repository,
array $visible_changesets) {
assert_instances_of($visible_changesets, 'DifferentialChangeset');
array $unfolded_changesets) {
assert_instances_of($unfolded_changesets, 'DifferentialChangeset');
$engine = PhabricatorSyntaxHighlighter::newEngine();
@ -889,7 +950,7 @@ final class DifferentialRevisionViewController
$sources);
$indexed_langs = array_fill_keys($langs, true);
foreach ($visible_changesets as $key => $changeset) {
foreach ($unfolded_changesets as $key => $changeset) {
$lang = $engine->getLanguageFromFilename($changeset->getFilename());
if (empty($indexed_langs) || isset($indexed_langs[$lang])) {
$symbol_indexes[$key] = array(

248
src/applications/differential/engine/DifferentialChangesetEngine.php Normal file
View file

@ -0,0 +1,248 @@
<?php
final class DifferentialChangesetEngine extends Phobject {
public function rebuildChangesets(array $changesets) {
assert_instances_of($changesets, 'DifferentialChangeset');
foreach ($changesets as $changeset) {
$this->detectGeneratedCode($changeset);
$this->computeHashes($changeset);
}
$this->detectCopiedCode($changesets);
}
/* -( Generated Code )----------------------------------------------------- */
private function detectGeneratedCode(DifferentialChangeset $changeset) {
$is_generated_trusted = $this->isTrustedGeneratedCode($changeset);
if ($is_generated_trusted) {
$changeset->setTrustedChangesetAttribute(
DifferentialChangeset::ATTRIBUTE_GENERATED,
$is_generated_trusted);
}
$is_generated_untrusted = $this->isUntrustedGeneratedCode($changeset);
if ($is_generated_untrusted) {
$changeset->setUntrustedChangesetAttribute(
DifferentialChangeset::ATTRIBUTE_GENERATED,
$is_generated_untrusted);
}
}
private function isTrustedGeneratedCode(DifferentialChangeset $changeset) {
$filename = $changeset->getFilename();
$paths = PhabricatorEnv::getEnvConfig('differential.generated-paths');
foreach ($paths as $regexp) {
if (preg_match($regexp, $filename)) {
return true;
}
}
return false;
}
private function isUntrustedGeneratedCode(DifferentialChangeset $changeset) {
if ($changeset->getHunks()) {
$new_data = $changeset->makeNewFile();
if (strpos($new_data, '@'.'generated') !== false) {
return true;
}
}
return false;
}
/* -( Content Hashes )----------------------------------------------------- */
private function computeHashes(DifferentialChangeset $changeset) {
$effect_key = DifferentialChangeset::METADATA_EFFECT_HASH;
$effect_hash = $this->newEffectHash($changeset);
if ($effect_hash !== null) {
$changeset->setChangesetMetadata($effect_key, $effect_hash);
}
}
private function newEffectHash(DifferentialChangeset $changeset) {
if ($changeset->getHunks()) {
$new_data = $changeset->makeNewFile();
return PhabricatorHash::digestForIndex($new_data);
}
return null;
}
/* -( Copied Code )-------------------------------------------------------- */
private function detectCopiedCode(array $changesets) {
$min_width = 30;
$min_lines = 3;
$map = array();
$files = array();
$types = array();
foreach ($changesets as $changeset) {
$file = $changeset->getFilename();
foreach ($changeset->getHunks() as $hunk) {
$lines = $hunk->getStructuredOldFile();
foreach ($lines as $line => $info) {
$type = $info['type'];
if ($type == '\\') {
continue;
}
$types[$file][$line] = $type;
$text = $info['text'];
$text = trim($text);
$files[$file][$line] = $text;
if (strlen($text) >= $min_width) {
$map[$text][] = array($file, $line);
}
}
}
}
foreach ($changesets as $changeset) {
$copies = array();
foreach ($changeset->getHunks() as $hunk) {
$added = $hunk->getStructuredNewFile();
$atype = array();
foreach ($added as $line => $info) {
$atype[$line] = $info['type'];
$added[$line] = trim($info['text']);
}
$skip_lines = 0;
foreach ($added as $line => $code) {
if ($skip_lines) {
// We're skipping lines that we already processed because we
// extended a block above them downward to include them.
$skip_lines--;
continue;
}
if ($atype[$line] !== '+') {
// This line hasn't been changed in the new file, so don't try
// to figure out where it came from.
continue;
}
if (empty($map[$code])) {
// This line was too short to trigger copy/move detection.
continue;
}
if (count($map[$code]) > 16) {
// If there are a large number of identical lines in this diff,
// don't try to figure out where this block came from: the analysis
// is O(N^2), since we need to compare every line against every
// other line. Even if we arrive at a result, it is unlikely to be
// meaningful. See T5041.
continue;
}
$best_length = 0;
// Explore all candidates.
foreach ($map[$code] as $val) {
list($file, $orig_line) = $val;
$length = 1;
// Search backward and forward to find all of the adjacent lines
// which match.
foreach (array(-1, 1) as $direction) {
$offset = $direction;
while (true) {
if (isset($copies[$line + $offset])) {
// If we run into a block above us which we've already
// attributed to a move or copy from elsewhere, stop
// looking.
break;
}
if (!isset($added[$line + $offset])) {
// If we've run off the beginning or end of the new file,
// stop looking.
break;
}
if (!isset($files[$file][$orig_line + $offset])) {
// If we've run off the beginning or end of the original
// file, we also stop looking.
break;
}
$old = $files[$file][$orig_line + $offset];
$new = $added[$line + $offset];
if ($old !== $new) {
// If the old line doesn't match the new line, stop
// looking.
break;
}
$length++;
$offset += $direction;
}
}
if ($length < $best_length) {
// If we already know of a better source (more matching lines)
// for this move/copy, stick with that one. We prefer long
// copies/moves which match a lot of context over short ones.
continue;
}
if ($length == $best_length) {
if (idx($types[$file], $orig_line) != '-') {
// If we already know of an equally good source (same number
// of matching lines) and this isn't a move, stick with the
// other one. We prefer moves over copies.
continue;
}
}
$best_length = $length;
// ($offset - 1) contains number of forward matching lines.
$best_offset = $offset - 1;
$best_file = $file;
$best_line = $orig_line;
}
$file = ($best_file == $changeset->getFilename() ? '' : $best_file);
for ($i = $best_length; $i--; ) {
$type = idx($types[$best_file], $best_line + $best_offset - $i);
$copies[$line + $best_offset - $i] = ($best_length < $min_lines
? array() // Ignore short blocks.
: array($file, $best_line + $best_offset - $i, $type));
}
$skip_lines = $best_offset;
}
}
$copies = array_filter($copies);
if ($copies) {
$metadata = $changeset->getMetadata();
$metadata['copy:lines'] = $copies;
$changeset->setMetadata($metadata);
}
}
}
}

96
src/applications/differential/management/PhabricatorDifferentialRebuildChangesetsWorkflow.php Normal file
View file

@ -0,0 +1,96 @@
<?php
final class PhabricatorDifferentialRebuildChangesetsWorkflow
extends PhabricatorDifferentialManagementWorkflow {
protected function didConstruct() {
$this
->setName('rebuild-changesets')
->setExamples('**rebuild-changesets** --revision __revision__')
->setSynopsis(pht('Rebuild changesets for a revision.'))
->setArguments(
array(
array(
'name' => 'revision',
'param' => 'revision',
'help' => pht('Revision to rebuild changesets for.'),
),
));
}
public function execute(PhutilArgumentParser $args) {
$viewer = $this->getViewer();
$revision_identifier = $args->getArg('revision');
if (!$revision_identifier) {
throw new PhutilArgumentUsageException(
pht('Specify a revision to rebuild changesets for with "--revision".'));
}
$revision = id(new PhabricatorObjectQuery())
->setViewer($viewer)
->withNames(array($revision_identifier))
->executeOne();
if ($revision) {
if (!($revision instanceof DifferentialRevision)) {
throw new PhutilArgumentUsageException(
pht(
'Object "%s" specified by "--revision" must be a Differential '.
'revision.'));
}
} else {
$revision = id(new DifferentialRevisionQuery())
->setViewer($viewer)
->withIDs(array($revision_identifier))
->executeOne();
}
if (!$revision) {
throw new PhutilArgumentUsageException(
pht(
'No revision "%s" exists.',
$revision_identifier));
}
$diffs = id(new DifferentialDiffQuery())
->setViewer($viewer)
->withRevisionIDs(array($revision->getID()))
->execute();
$changesets = id(new DifferentialChangesetQuery())
->setViewer($viewer)
->withDiffs($diffs)
->needHunks(true)
->execute();
$changeset_groups = mgroup($changesets, 'getDiffID');
foreach ($changeset_groups as $diff_id => $changesets) {
echo tsprintf(
"%s\n",
pht(
'Rebuilding %s changeset(s) for diff ID %d.',
phutil_count($changesets),
$diff_id));
foreach ($changesets as $changeset) {
echo tsprintf(
" %s\n",
$changeset->getFilename());
}
id(new DifferentialChangesetEngine())
->rebuildChangesets($changesets);
foreach ($changesets as $changeset) {
$changeset->save();
}
echo tsprintf(
"%s\n",
pht('Done.'));
}
}
}

161
src/applications/differential/parser/DifferentialChangesetParser.php
View file

@ -1419,167 +1419,6 @@ final class DifferentialChangesetParser extends Phobject {
return sprintf('%d%%', 100 * ($covered / ($covered + $not_covered)));
}
public function detectCopiedCode(
array $changesets,
$min_width = 30,
$min_lines = 3) {
assert_instances_of($changesets, 'DifferentialChangeset');
$map = array();
$files = array();
$types = array();
foreach ($changesets as $changeset) {
$file = $changeset->getFilename();
foreach ($changeset->getHunks() as $hunk) {
$lines = $hunk->getStructuredOldFile();
foreach ($lines as $line => $info) {
$type = $info['type'];
if ($type == '\\') {
continue;
}
$types[$file][$line] = $type;
$text = $info['text'];
$text = trim($text);
$files[$file][$line] = $text;
if (strlen($text) >= $min_width) {
$map[$text][] = array($file, $line);
}
}
}
}
foreach ($changesets as $changeset) {
$copies = array();
foreach ($changeset->getHunks() as $hunk) {
$added = $hunk->getStructuredNewFile();
$atype = array();
foreach ($added as $line => $info) {
$atype[$line] = $info['type'];
$added[$line] = trim($info['text']);
}
$skip_lines = 0;
foreach ($added as $line => $code) {
if ($skip_lines) {
// We're skipping lines that we already processed because we
// extended a block above them downward to include them.
$skip_lines--;
continue;
}
if ($atype[$line] !== '+') {
// This line hasn't been changed in the new file, so don't try
// to figure out where it came from.
continue;
}
if (empty($map[$code])) {
// This line was too short to trigger copy/move detection.
continue;
}
if (count($map[$code]) > 16) {
// If there are a large number of identical lines in this diff,
// don't try to figure out where this block came from: the analysis
// is O(N^2), since we need to compare every line against every
// other line. Even if we arrive at a result, it is unlikely to be
// meaningful. See T5041.
continue;
}
$best_length = 0;
// Explore all candidates.
foreach ($map[$code] as $val) {
list($file, $orig_line) = $val;
$length = 1;
// Search backward and forward to find all of the adjacent lines
// which match.
foreach (array(-1, 1) as $direction) {
$offset = $direction;
while (true) {
if (isset($copies[$line + $offset])) {
// If we run into a block above us which we've already
// attributed to a move or copy from elsewhere, stop
// looking.
break;
}
if (!isset($added[$line + $offset])) {
// If we've run off the beginning or end of the new file,
// stop looking.
break;
}
if (!isset($files[$file][$orig_line + $offset])) {
// If we've run off the beginning or end of the original
// file, we also stop looking.
break;
}
$old = $files[$file][$orig_line + $offset];
$new = $added[$line + $offset];
if ($old !== $new) {
// If the old line doesn't match the new line, stop
// looking.
break;
}
$length++;
$offset += $direction;
}
}
if ($length < $best_length) {
// If we already know of a better source (more matching lines)
// for this move/copy, stick with that one. We prefer long
// copies/moves which match a lot of context over short ones.
continue;
}
if ($length == $best_length) {
if (idx($types[$file], $orig_line) != '-') {
// If we already know of an equally good source (same number
// of matching lines) and this isn't a move, stick with the
// other one. We prefer moves over copies.
continue;
}
}
$best_length = $length;
// ($offset - 1) contains number of forward matching lines.
$best_offset = $offset - 1;
$best_file = $file;
$best_line = $orig_line;
}
$file = ($best_file == $changeset->getFilename() ? '' : $best_file);
for ($i = $best_length; $i--; ) {
$type = idx($types[$best_file], $best_line + $best_offset - $i);
$copies[$line + $best_offset - $i] = ($best_length < $min_lines
? array() // Ignore short blocks.
: array($file, $best_line + $best_offset - $i, $type));
}
$skip_lines = $best_offset;
}
}
$copies = array_filter($copies);
if ($copies) {
$metadata = $changeset->getMetadata();
$metadata['copy:lines'] = $copies;
$changeset->setMetadata($metadata);
}
}
return $changesets;
}
/**
* Build maps from lines comments appear on to actual lines.
*/

43
src/applications/differential/storage/DifferentialChangeset.php
View file

@ -26,6 +26,7 @@ final class DifferentialChangeset
const METADATA_TRUSTED_ATTRIBUTES = 'attributes.trusted';
const METADATA_UNTRUSTED_ATTRIBUTES = 'attributes.untrusted';
const METADATA_EFFECT_HASH = 'hash.effect';
const ATTRIBUTE_GENERATED = 'generated';
@ -143,6 +144,48 @@ final class DifferentialChangeset
return $ret;
}
/**
* Test if this changeset and some other changeset put the affected file in
* the same state.
*
* @param DifferentialChangeset Changeset to compare against.
* @return bool True if the two changesets have the same effect.
*/
public function hasSameEffectAs(DifferentialChangeset $other) {
if ($this->getFilename() !== $other->getFilename()) {
return false;
}
$hash_key = self::METADATA_EFFECT_HASH;
$u_hash = $this->getChangesetMetadata($hash_key);
if ($u_hash === null) {
return false;
}
$v_hash = $other->getChangesetMetadata($hash_key);
if ($v_hash === null) {
return false;
}
if ($u_hash !== $v_hash) {
return false;
}
// Make sure the final states for the file properties (like the "+x"
// executable bit) match one another.
$u_props = $this->getNewProperties();
$v_props = $other->getNewProperties();
ksort($u_props);
ksort($v_props);
if ($u_props !== $v_props) {
return false;
}
return true;
}
public function getSortKey() {
$sort_key = $this->getFilename();
// Sort files with ".h" in them first, so headers (.h, .hpp) come before

59
src/applications/differential/storage/DifferentialDiff.php
View file

@ -226,23 +226,18 @@ final class DifferentialDiff
$changeset->setAddLines($add_lines);
$changeset->setDelLines($del_lines);
self::detectGeneratedCode($changeset);
$diff->addUnsavedChangeset($changeset);
}
$diff->setLineCount($lines);
$parser = new DifferentialChangesetParser();
$changesets = $parser->detectCopiedCode(
$diff->getChangesets(),
$min_width = 30,
$min_lines = 3);
$diff->attachChangesets($changesets);
$changesets = $diff->getChangesets();
id(new DifferentialChangesetEngine())
->rebuildChangesets($changesets);
return $diff;
}
public function getDiffDict() {
$dict = array(
'id' => $this->getID(),
@ -823,50 +818,4 @@ final class DifferentialDiff
);
}
private static function detectGeneratedCode(
DifferentialChangeset $changeset) {
$is_generated_trusted = self::isTrustedGeneratedCode($changeset);
if ($is_generated_trusted) {
$changeset->setTrustedChangesetAttribute(
DifferentialChangeset::ATTRIBUTE_GENERATED,
$is_generated_trusted);
}
$is_generated_untrusted = self::isUntrustedGeneratedCode($changeset);
if ($is_generated_untrusted) {
$changeset->setUntrustedChangesetAttribute(
DifferentialChangeset::ATTRIBUTE_GENERATED,
$is_generated_untrusted);
}
}
private static function isTrustedGeneratedCode(
DifferentialChangeset $changeset) {
$filename = $changeset->getFilename();
$paths = PhabricatorEnv::getEnvConfig('differential.generated-paths');
foreach ($paths as $regexp) {
if (preg_match($regexp, $filename)) {
return true;
}
}
return false;
}
private static function isUntrustedGeneratedCode(
DifferentialChangeset $changeset) {
if ($changeset->getHunks()) {
$new_data = $changeset->makeNewFile();
if (strpos($new_data, '@'.'generated') !== false) {
return true;
}
}
return false;
}
}

26
src/applications/diffusion/engine/DiffusionCommitHookEngine.php
View file

@ -37,6 +37,7 @@ final class DiffusionCommitHookEngine extends Phobject {
private $rejectDetails;
private $emailPHIDs = array();
private $changesets = array();
private $changesetsSize = 0;
/* -( Config )------------------------------------------------------------- */
@ -1121,11 +1122,22 @@ final class DiffusionCommitHookEngine extends Phobject {
return;
}
// See T13142. Don't cache more than 64MB of changesets. For normal small
// pushes, caching everything here can let us hit the cache from Herald if
// we need to run content rules, which speeds things up a bit. For large
// pushes, we may not be able to hold everything in memory.
$cache_limit = 1024 * 1024 * 64;
foreach ($content_updates as $update) {
$identifier = $update->getRefNew();
try {
$changesets = $this->loadChangesetsForCommit($identifier);
$this->changesets[$identifier] = $changesets;
$info = $this->loadChangesetsForCommit($identifier);
list($changesets, $size) = $info;
if ($this->changesetsSize + $size <= $cache_limit) {
$this->changesets[$identifier] = $changesets;
$this->changesetsSize += $size;
}
} catch (Exception $ex) {
$this->changesets[$identifier] = $ex;
@ -1207,7 +1219,11 @@ final class DiffusionCommitHookEngine extends Phobject {
$changes = $parser->parseDiff($raw_diff);
$diff = DifferentialDiff::newEphemeralFromRawChanges(
$changes);
return $diff->getChangesets();
$changesets = $diff->getChangesets();
$size = strlen($raw_diff);
return array($changesets, $size);
}
public function getChangesetsForCommit($identifier) {
@ -1221,7 +1237,9 @@ final class DiffusionCommitHookEngine extends Phobject {
return $cached;
}
return $this->loadChangesetsForCommit($identifier);
$info = $this->loadChangesetsForCommit($identifier);
list($changesets, $size) = $info;
return $changesets;
}
public function loadCommitRefForCommit($identifier) {

2
src/applications/diffusion/herald/HeraldCommitAdapter.php
View file

@ -207,7 +207,7 @@ final class HeraldCommitAdapter
}
public static function getEnormousByteLimit() {
return 1024 * 1024 * 1024; // 1GB
return 256 * 1024 * 1024; // 256MB. See T13142 and T13143.
}
public static function getEnormousTimeLimit() {

27
src/applications/diffusion/protocol/DiffusionSubversionWireProtocol.php
View file

@ -33,7 +33,24 @@ final class DiffusionSubversionWireProtocol extends Phobject {
$messages = array();
while (true) {
if ($this->state == 'item') {
if ($this->state == 'space') {
// Consume zero or more extra spaces after matching an item. The
// protocol requires at least one space, but allows more than one.
$matches = null;
if (!preg_match('/^(\s*)\S/', $this->buffer, $matches)) {
// Wait for more data.
break;
}
// We have zero or more spaces and then some other character, so throw
// the spaces away and continue parsing frames.
if (strlen($matches[1])) {
$this->buffer = substr($this->buffer, strlen($matches[1]));
}
$this->state = 'item';
} else if ($this->state == 'item') {
$match = null;
$result = null;
$buf = $this->buffer;
@ -69,6 +86,12 @@ final class DiffusionSubversionWireProtocol extends Phobject {
);
$this->raw = '';
}
// Consume any extra whitespace after an item. If we're in the
// "bytes" state, we aren't looking for whitespace.
if ($this->state == 'item') {
$this->state = 'space';
}
} else {
// No matches yet, wait for more data.
break;
@ -90,7 +113,7 @@ final class DiffusionSubversionWireProtocol extends Phobject {
// Strip off the terminal space.
$this->pushItem(substr($this->byteBuffer, 0, -1), 'string');
$this->byteBuffer = '';
$this->state = 'item';
$this->state = 'space';
}
} else {
throw new Exception(pht("Invalid state '%s'!", $this->state));

59
src/applications/diffusion/protocol/__tests__/DiffusionSubversionWireProtocolTestCase.php
View file

@ -59,6 +59,50 @@ final class DiffusionSubversionWireProtocolTestCase
),
),
));
// This is testing that multiple spaces are parsed correctly. See T13140
// for discussion.
$this->assertSameSubversionMessages(
'( get-file true false ) ',
// ^-- Note extra space!
array(
array(
array(
'type' => 'word',
'value' => 'get-file',
),
array(
'type' => 'word',
'value' => 'true',
),
array(
'type' => 'word',
'value' => 'false',
),
),
),
'( get-file true false ) ');
$this->assertSameSubversionMessages(
'( duck 5:quack moo ) ',
array(
array(
array(
'type' => 'word',
'value' => 'duck',
),
array(
'type' => 'string',
'value' => 'quack',
),
array(
'type' => 'word',
'value' => 'moo',
),
),
),
'( duck 5:quack moo ) ');
}
public function testSubversionWireProtocolPartialFrame() {
@ -86,7 +130,11 @@ final class DiffusionSubversionWireProtocolTestCase
ipull($msg2, 'structure'));
}
private function assertSameSubversionMessages($string, array $structs) {
private function assertSameSubversionMessages(
$string,
array $structs,
$serial_string = null) {
$proto = new DiffusionSubversionWireProtocol();
// Verify that the wire message parses into the structs.
@ -100,6 +148,13 @@ final class DiffusionSubversionWireProtocolTestCase
$serial[] = $proto->serializeStruct($struct);
}
$serial = implode('', $serial);
$this->assertEqual($string, $serial, 'serialize<'.$string.'>');
if ($serial_string === null) {
$expect_serial = $string;
} else {
$expect_serial = $serial_string;
}
$this->assertEqual($expect_serial, $serial, 'serialize<'.$string.'>');
}
}

5
src/applications/people/controller/PhabricatorPeopleNewController.php
View file

@ -7,6 +7,11 @@ final class PhabricatorPeopleNewController
$type = $request->getURIData('type');
$admin = $request->getUser();
id(new PhabricatorAuthSessionEngine())->requireHighSecuritySession(
$admin,
$request,
$this->getApplicationURI());
$is_bot = false;
$is_list = false;
switch ($type) {

7
src/infrastructure/cluster/PhabricatorDatabaseRef.php
View file

@ -29,6 +29,7 @@ final class PhabricatorDatabaseRef
private $connectionLatency;
private $connectionStatus;
private $connectionMessage;
private $connectionException;
private $replicaStatus;
private $replicaMessage;
@ -453,6 +454,7 @@ final class PhabricatorDatabaseRef
return false;
}
$this->connectionException = null;
try {
$connection->openConnection();
$reachable = true;
@ -463,6 +465,7 @@ final class PhabricatorDatabaseRef
// yet.
throw $ex;
} catch (Exception $ex) {
$this->connectionException = $ex;
$reachable = false;
}
@ -506,6 +509,10 @@ final class PhabricatorDatabaseRef
return $this->healthRecord;
}
public function getConnectionException() {
return $this->connectionException;
}
public static function getActiveDatabaseRefs() {
$refs = array();

29
src/infrastructure/storage/lisk/PhabricatorLiskDAO.php
View file

@ -80,6 +80,8 @@ abstract class PhabricatorLiskDAO extends LiskDAO {
$master = PhabricatorDatabaseRef::getMasterDatabaseRefForApplication(
$application);
$master_exception = null;
if ($master && !$master->isSevered()) {
$connection = $master->newApplicationConnection($database);
if ($master->isReachable($connection)) {
@ -91,6 +93,8 @@ abstract class PhabricatorLiskDAO extends LiskDAO {
PhabricatorEnv::setReadOnly(
true,
PhabricatorEnv::READONLY_UNREACHABLE);
$master_exception = $master->getConnectionException();
}
}
@ -108,7 +112,7 @@ abstract class PhabricatorLiskDAO extends LiskDAO {
$this->raiseUnconfigured($database);
}
$this->raiseUnreachable($database);
$this->raiseUnreachable($database, $master_exception);
}
private function raiseImproperWrite($database) {
@ -136,13 +140,22 @@ abstract class PhabricatorLiskDAO extends LiskDAO {
$database));
}
private function raiseUnreachable($database) {
throw new PhabricatorClusterStrandedException(
pht(
'Unable to establish a connection to any database host '.
'(while trying "%s"). All masters and replicas are completely '.
'unreachable.',
$database));
private function raiseUnreachable($database, Exception $proxy = null) {
$message = pht(
'Unable to establish a connection to any database host '.
'(while trying "%s"). All masters and replicas are completely '.
'unreachable.',
$database);
if ($proxy) {
$proxy_message = pht(
'%s: %s',
get_class($proxy),
$proxy->getMessage());
$message = $message."\n\n".$proxy_message;
}
throw new PhabricatorClusterStrandedException($message);
}

41
src/infrastructure/storage/lisk/PhabricatorQueryIterator.php Normal file
View file

@ -0,0 +1,41 @@
<?php
final class PhabricatorQueryIterator extends PhutilBufferedIterator {
private $query;
private $pager;
public function __construct(PhabricatorCursorPagedPolicyAwareQuery $query) {
$this->query = $query;
}
protected function didRewind() {
$this->pager = new AphrontCursorPagerView();
}
public function key() {
return $this->current()->getID();
}
protected function loadPage() {
if (!$this->pager) {
return array();
}
$pager = clone $this->pager;
$query = clone $this->query;
$results = $query->executeWithCursorPager($pager);
// If we got less than a full page of results, this was the last set of
// results. Throw away the pager so we end iteration.
if (count($results) < $pager->getPageSize()) {
$this->pager = null;
} else {
$this->pager->setAfterID($pager->getNextPageID());
}
return $results;
}
}