Prevent enormous changes from being pushed to repositoires by default

Summary:
Fixes T13031. "Enormous" changes are basically changes which are too large to hold in memory, although the actual definition we use today is "more than 1GB of change text or `git diff` runs for more than 15 minutes".

If an install configures a Herald content rule like "when content matches /XYZ/, do something" and then a user pushes a 30 GB source file, we can't put it into memory to `preg_match()` it. Currently, the way to handle this case is to write a separate Herald rule that rejects enormous changes. However, this isn't obvious and means the default behavior is unsafe.

Make the default behavior safe by rejecting these changes with a message, similar to how we reject "dangerous" changes (which permanently delete or overwrite history) by default.

Also, change a couple of UI strings from "Enormous" to "Very Large" to reduce ambiguity. See <https://discourse.phabricator-community.org/t/herald-enormous-check/822>.

Test Plan: Changed the definition of "enormous" from 1GB to 1 byte. Pushed a change; got rejected. Allowed enormous changes, pushed, got rejected by a Herald rule. Disabled the Herald rule, pushed, got a clean push. Prevented enormous changes again. Grepped for "enormous" elsewhere in the UI.

Reviewers: amckinley

Reviewed By: amckinley

Subscribers: joshuaspence

Maniphest Tasks: T13031

Differential Revision: https://secure.phabricator.com/D18850

src/__phutil_library_map__.php

@@ -860,6 +860,7 @@ phutil_register_library_map(array(
     'DiffusionRepositoryEditDangerousController' => 'applications/diffusion/controller/DiffusionRepositoryEditDangerousController.php',
     'DiffusionRepositoryEditDeleteController' => 'applications/diffusion/controller/DiffusionRepositoryEditDeleteController.php',
     'DiffusionRepositoryEditEngine' => 'applications/diffusion/editor/DiffusionRepositoryEditEngine.php',
+    'DiffusionRepositoryEditEnormousController' => 'applications/diffusion/controller/DiffusionRepositoryEditEnormousController.php',
     'DiffusionRepositoryEditUpdateController' => 'applications/diffusion/controller/DiffusionRepositoryEditUpdateController.php',
     'DiffusionRepositoryFunctionDatasource' => 'applications/diffusion/typeahead/DiffusionRepositoryFunctionDatasource.php',
     'DiffusionRepositoryHistoryManagementPanel' => 'applications/diffusion/management/DiffusionRepositoryHistoryManagementPanel.php',
@@ -5923,6 +5924,7 @@ phutil_register_library_map(array(
     'DiffusionRepositoryEditDangerousController' => 'DiffusionRepositoryManageController',
     'DiffusionRepositoryEditDeleteController' => 'DiffusionRepositoryManageController',
     'DiffusionRepositoryEditEngine' => 'PhabricatorEditEngine',
+    'DiffusionRepositoryEditEnormousController' => 'DiffusionRepositoryManageController',
     'DiffusionRepositoryEditUpdateController' => 'DiffusionRepositoryManageController',
     'DiffusionRepositoryFunctionDatasource' => 'PhabricatorTypeaheadCompositeDatasource',
     'DiffusionRepositoryHistoryManagementPanel' => 'DiffusionRepositoryManagementPanel',

src/applications/diffusion/application/PhabricatorDiffusionApplication.php

@@ -89,6 +89,7 @@ final class PhabricatorDiffusionApplication extends PhabricatorApplication {
         'edit/' => array(
           'activate/' => 'DiffusionRepositoryEditActivateController',
           'dangerous/' => 'DiffusionRepositoryEditDangerousController',
+          'enormous/' => 'DiffusionRepositoryEditEnormousController',
           'delete/' => 'DiffusionRepositoryEditDeleteController',
           'update/' => 'DiffusionRepositoryEditUpdateController',
           'testautomation/' => 'DiffusionRepositoryTestAutomationController',

src/applications/diffusion/controller/DiffusionCommitController.php

@@ -277,9 +277,9 @@ final class DiffusionCommitController extends DiffusionController {
           'This commit is empty and does not affect any paths.'));
     } else if ($was_limited) {
       $info_panel = $this->renderStatusMessage(
-        pht('Enormous Commit'),
+        pht('Very Large Commit'),
         pht(
-          'This commit is enormous, and affects more than %d files. '.
+          'This commit is very large, and affects more than %d files. '.
           'Changes are not shown.',
           $hard_limit));
     } else if (!$this->getCommitExists()) {

src/applications/diffusion/controller/DiffusionRepositoryEditEnormousController.php

@@ -0,0 +1,90 @@
+<?php
+
+final class DiffusionRepositoryEditEnormousController
+  extends DiffusionRepositoryManageController {
+
+  public function handleRequest(AphrontRequest $request) {
+    $response = $this->loadDiffusionContextForEdit();
+    if ($response) {
+      return $response;
+    }
+
+    $viewer = $this->getViewer();
+    $drequest = $this->getDiffusionRequest();
+    $repository = $drequest->getRepository();
+
+    $panel_uri = id(new DiffusionRepositoryBasicsManagementPanel())
+      ->setRepository($repository)
+      ->getPanelURI();
+
+    if (!$repository->canAllowEnormousChanges()) {
+      return $this->newDialog()
+        ->setTitle(pht('Unprotectable Repository'))
+        ->appendParagraph(
+          pht(
+            'This repository can not be protected from enormous changes '.
+            'because Phabricator does not control what users are allowed '.
+            'to push to it.'))
+        ->addCancelButton($panel_uri);
+    }
+
+    if ($request->isFormPost()) {
+      $xaction = id(new PhabricatorRepositoryTransaction())
+        ->setTransactionType(PhabricatorRepositoryTransaction::TYPE_ENORMOUS)
+        ->setNewValue(!$repository->shouldAllowEnormousChanges());
+
+      $editor = id(new PhabricatorRepositoryEditor())
+        ->setContinueOnNoEffect(true)
+        ->setContentSourceFromRequest($request)
+        ->setActor($viewer)
+        ->applyTransactions($repository, array($xaction));
+
+      return id(new AphrontReloadResponse())->setURI($panel_uri);
+    }
+
+    if ($repository->shouldAllowEnormousChanges()) {
+      $title = pht('Prevent Enormous Changes');
+
+      $body = pht(
+        'It will no longer be possible to push enormous changes to this '.
+        'repository.');
+
+      $submit = pht('Prevent Enormous Changes');
+    } else {
+      $title = pht('Allow Enormous Changes');
+
+      $body = array(
+        pht(
+          'If you allow enormous changes, users can push commits which are '.
+          'too large for Herald to process content rules for. This can allow '.
+          'users to evade content rules implemented in Herald.'),
+        pht(
+          'You can selectively configure Herald by adding rules to prevent a '.
+          'subset of enormous changes (for example, based on who is trying '.
+          'to push the change).'),
+      );
+
+      $submit = pht('Allow Enormous Changes');
+    }
+
+    $more_help = pht(
+      'Enormous changes are commits which are too large to process with '.
+      'content rules because: the diff text for the change is larger than '.
+      '%s bytes; or the diff text takes more than %s seconds to extract.',
+      new PhutilNumber(HeraldCommitAdapter::getEnormousByteLimit()),
+      new PhutilNumber(HeraldCommitAdapter::getEnormousTimeLimit()));
+
+    $response = $this->newDialog();
+
+      foreach ((array)$body as $paragraph) {
+        $response->appendParagraph($paragraph);
+      }
+
+    return $response
+      ->setTitle($title)
+      ->appendParagraph($more_help)
+      ->addSubmitButton($submit)
+      ->addCancelButton($panel_uri);
+  }
+
+}

src/applications/diffusion/editor/DiffusionRepositoryEditEngine.php

@@ -309,6 +309,19 @@ final class DiffusionRepositoryEditEngine
         ->setConduitDescription(pht('Allow or prevent dangerous changes.'))
         ->setConduitTypeDescription(pht('New protection setting.'))
         ->setValue($object->shouldAllowDangerousChanges()),
+      id(new PhabricatorBoolEditField())
+        ->setKey('allowEnormousChanges')
+        ->setLabel(pht('Allow Enormous Changes'))
+        ->setIsCopyable(true)
+        ->setIsConduitOnly(true)
+        ->setOptions(
+          pht('Prevent Enormous Changes'),
+          pht('Allow Enormous Changes'))
+        ->setTransactionType(PhabricatorRepositoryTransaction::TYPE_ENORMOUS)
+        ->setDescription(pht('Permit enomrous changes to be made.'))
+        ->setConduitDescription(pht('Allow or prevent enormous changes.'))
+        ->setConduitTypeDescription(pht('New protection setting.'))
+        ->setValue($object->shouldAllowEnormousChanges()),
       id(new PhabricatorSelectEditField())
         ->setKey('status')
         ->setLabel(pht('Status'))

src/applications/diffusion/engine/DiffusionCommitHookEngine.php

@@ -34,6 +34,7 @@ final class DiffusionCommitHookEngine extends Phobject {
   private $rejectCode = PhabricatorRepositoryPushLog::REJECT_BROKEN;
   private $rejectDetails;
   private $emailPHIDs = array();
+  private $changesets = array();
 
 
 /* -(  Config  )------------------------------------------------------------- */
@@ -131,6 +132,15 @@ final class DiffusionCommitHookEngine extends Phobject {
       $this->applyHeraldRefRules($ref_updates, $all_updates);
 
       $content_updates = $this->findContentUpdates($ref_updates);
+
+      try {
+        $this->rejectEnormousChanges($content_updates);
+      } catch (DiffusionCommitHookRejectException $ex) {
+        // If we're rejecting enormous changes, flag everything.
+        $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_ENORMOUS;
+        throw $ex;
+      }
+
       $all_updates = array_merge($all_updates, $content_updates);
 
       $this->applyHeraldContentRules($content_updates, $all_updates);
@@ -1079,7 +1089,37 @@ final class DiffusionCommitHookEngine extends Phobject {
       ->setEpoch(time());
   }
 
-  public function loadChangesetsForCommit($identifier) {
+  private function rejectEnormousChanges(array $content_updates) {
+    $repository = $this->getRepository();
+    if ($repository->shouldAllowEnormousChanges()) {
+      return;
+    }
+
+    foreach ($content_updates as $update) {
+      $identifier = $update->getRefNew();
+      try {
+        $changesets = $this->loadChangesetsForCommit($identifier);
+        $this->changesets[$identifier] = $changesets;
+      } catch (Exception $ex) {
+        $this->changesets[$identifier] = $ex;
+
+        $message = pht(
+          'ENORMOUS CHANGE'.
+          "\n".
+          'Enormous change protection is enabled for this repository, but '.
+          'you are pushing an enormous change ("%s"). Edit the repository '.
+          'configuration before making enormous changes.'.
+          "\n\n".
+          "Content Exception: %s",
+          $identifier,
+          $ex->getMessage());
+
+        throw new DiffusionCommitHookRejectException($message);
+      }
+    }
+  }
+
+  private function loadChangesetsForCommit($identifier) {
     $byte_limit = HeraldCommitAdapter::getEnormousByteLimit();
     $time_limit = HeraldCommitAdapter::getEnormousTimeLimit();
 
@@ -1126,9 +1166,10 @@ final class DiffusionCommitHookEngine extends Phobject {
     if (strlen($raw_diff) >= $byte_limit) {
       throw new Exception(
         pht(
-          'The raw text of this change is enormous (larger than %d '.
-          'bytes). Herald can not process it.',
-          $byte_limit));
+          'The raw text of this change ("%s") is enormous (larger than %s '.
+          'bytes).',
+          $identifier,
+          new PhutilNumber($byte_limit)));
     }
 
     if (!strlen($raw_diff)) {
@@ -1143,6 +1184,20 @@ final class DiffusionCommitHookEngine extends Phobject {
     return $diff->getChangesets();
   }
 
+  public function getChangesetsForCommit($identifier) {
+    if (isset($this->changesets[$identifier])) {
+      $cached = $this->changesets[$identifier];
+
+      if ($cached instanceof Exception) {
+        throw $cached;
+      }
+
+      return $cached;
+    }
+
+    return $this->loadChangesetsForCommit($identifier);
+  }
+
   public function loadCommitRefForCommit($identifier) {
     $repository = $this->getRepository();
     $vcs = $repository->getVersionControlSystem();

src/applications/diffusion/herald/HeraldPreCommitContentAdapter.php

@@ -37,7 +37,7 @@ final class HeraldPreCommitContentAdapter extends HeraldPreCommitAdapter {
   public function getDiffContent($type) {
     if ($this->changesets === null) {
       try {
-        $this->changesets = $this->getHookEngine()->loadChangesetsForCommit(
+        $this->changesets = $this->getHookEngine()->getChangesetsForCommit(
           $this->getObject()->getRefNew());
       } catch (Exception $ex) {
         $this->changesets = $ex;

src/applications/diffusion/management/DiffusionRepositoryBasicsManagementPanel.php

@@ -43,6 +43,7 @@ final class DiffusionRepositoryBasicsManagementPanel
     $delete_uri = $repository->getPathURI('edit/delete/');
     $encoding_uri = $this->getEditPageURI('encoding');
     $dangerous_uri = $repository->getPathURI('edit/dangerous/');
+    $enormous_uri = $repository->getPathURI('edit/enormous/');
 
     if ($repository->isTracked()) {
       $activate_label = pht('Deactivate Repository');
@@ -59,6 +60,15 @@ final class DiffusionRepositoryBasicsManagementPanel
       $can_dangerous = ($can_edit && $repository->canAllowDangerousChanges());
     }
 
+    $should_enormous = $repository->shouldAllowEnormousChanges();
+    if ($should_enormous) {
+      $enormous_name = pht('Prevent Enormous Changes');
+      $can_enormous = $can_edit;
+    } else {
+      $enormous_name = pht('Allow Enormous Changes');
+      $can_enormous = ($can_edit && $repository->canAllowEnormousChanges());
+    }
+
     $action_list->addAction(
       id(new PhabricatorActionView())
         ->setName(pht('Edit Basic Information'))
@@ -80,6 +90,13 @@ final class DiffusionRepositoryBasicsManagementPanel
         ->setDisabled(!$can_dangerous)
         ->setWorkflow(true));
 
+    $action_list->addAction(
+      id(new PhabricatorActionView())
+        ->setName($enormous_name)
+        ->setHref($enormous_uri)
+        ->setDisabled(!$can_enormous)
+        ->setWorkflow(true));
+
     $action_list->addAction(
       id(new PhabricatorActionView())
         ->setHref($activate_uri)
@@ -198,6 +215,20 @@ final class DiffusionRepositoryBasicsManagementPanel
 
     $view->addProperty(pht('Dangerous Changes'), $dangerous);
 
+    $can_enormous = $repository->canAllowEnormousChanges();
+    if (!$can_enormous) {
+      $enormous = phutil_tag('em', array(), pht('Not Preventable'));
+    } else {
+      $should_enormous = $repository->shouldAllowEnormousChanges();
+      if ($should_enormous) {
+        $enormous = pht('Allowed');
+      } else {
+        $enormous = pht('Not Allowed');
+      }
+    }
+
+    $view->addProperty(pht('Enormous Changes'), $enormous);
+
     return $view;
   }
 

src/applications/files/config/PhabricatorFilesConfigOptions.php

@@ -134,7 +134,7 @@ final class PhabricatorFilesConfigOptions
             "Configure which uploaded file types may be viewed directly ".
             "in the browser. Other file types will be downloaded instead ".
             "of displayed. This is mainly a usability consideration, since ".
-            "browsers tend to freak out when viewing enormous binary files.".
+            "browsers tend to freak out when viewing very large binary files.".
             "\n\n".
             "The keys in this map are viewable MIME types; the values are ".
             "the MIME types they are delivered as when they are viewed in ".

src/applications/repository/editor/PhabricatorRepositoryEditor.php

@@ -28,6 +28,7 @@ final class PhabricatorRepositoryEditor
     $types[] = PhabricatorRepositoryTransaction::TYPE_AUTOCLOSE;
     $types[] = PhabricatorRepositoryTransaction::TYPE_PUSH_POLICY;
     $types[] = PhabricatorRepositoryTransaction::TYPE_DANGEROUS;
+    $types[] = PhabricatorRepositoryTransaction::TYPE_ENORMOUS;
     $types[] = PhabricatorRepositoryTransaction::TYPE_SLUG;
     $types[] = PhabricatorRepositoryTransaction::TYPE_SERVICE;
     $types[] = PhabricatorRepositoryTransaction::TYPE_SYMBOLS_LANGUAGE;
@@ -76,6 +77,8 @@ final class PhabricatorRepositoryEditor
         return $object->getPushPolicy();
       case PhabricatorRepositoryTransaction::TYPE_DANGEROUS:
         return $object->shouldAllowDangerousChanges();
+      case PhabricatorRepositoryTransaction::TYPE_ENORMOUS:
+        return $object->shouldAllowEnormousChanges();
       case PhabricatorRepositoryTransaction::TYPE_SLUG:
         return $object->getRepositorySlug();
       case PhabricatorRepositoryTransaction::TYPE_SERVICE:
@@ -110,6 +113,7 @@ final class PhabricatorRepositoryEditor
       case PhabricatorRepositoryTransaction::TYPE_VCS:
       case PhabricatorRepositoryTransaction::TYPE_PUSH_POLICY:
       case PhabricatorRepositoryTransaction::TYPE_DANGEROUS:
+      case PhabricatorRepositoryTransaction::TYPE_ENORMOUS:
       case PhabricatorRepositoryTransaction::TYPE_SERVICE:
       case PhabricatorRepositoryTransaction::TYPE_SYMBOLS_LANGUAGE:
       case PhabricatorRepositoryTransaction::TYPE_SYMBOLS_SOURCES:
@@ -184,6 +188,9 @@ final class PhabricatorRepositoryEditor
       case PhabricatorRepositoryTransaction::TYPE_DANGEROUS:
         $object->setDetail('allow-dangerous-changes', $xaction->getNewValue());
         return;
+      case PhabricatorRepositoryTransaction::TYPE_ENORMOUS:
+        $object->setDetail('allow-enormous-changes', $xaction->getNewValue());
+        return;
       case PhabricatorRepositoryTransaction::TYPE_SLUG:
         $object->setRepositorySlug($xaction->getNewValue());
         return;
@@ -248,6 +255,7 @@ final class PhabricatorRepositoryEditor
       case PhabricatorRepositoryTransaction::TYPE_AUTOCLOSE:
       case PhabricatorRepositoryTransaction::TYPE_PUSH_POLICY:
       case PhabricatorRepositoryTransaction::TYPE_DANGEROUS:
+      case PhabricatorRepositoryTransaction::TYPE_ENORMOUS:
       case PhabricatorRepositoryTransaction::TYPE_SLUG:
       case PhabricatorRepositoryTransaction::TYPE_SERVICE:
       case PhabricatorRepositoryTransaction::TYPE_SYMBOLS_SOURCES:

src/applications/repository/storage/PhabricatorRepository.php

@@ -1672,6 +1672,18 @@ final class PhabricatorRepository extends PhabricatorRepositoryDAO
     return (bool)$this->getDetail('allow-dangerous-changes');
   }
 
+  public function canAllowEnormousChanges() {
+    if (!$this->isHosted()) {
+      return false;
+    }
+
+    return true;
+  }
+
+  public function shouldAllowEnormousChanges() {
+    return (bool)$this->getDetail('allow-enormous-changes');
+  }
+
   public function writeStatusMessage(
     $status_type,
     $status_code,

src/applications/repository/storage/PhabricatorRepositoryPushLog.php

@@ -23,12 +23,14 @@ final class PhabricatorRepositoryPushLog
   const CHANGEFLAG_APPEND = 4;
   const CHANGEFLAG_REWRITE = 8;
   const CHANGEFLAG_DANGEROUS = 16;
+  const CHANGEFLAG_ENORMOUS = 32;
 
   const REJECT_ACCEPT = 0;
   const REJECT_DANGEROUS = 1;
   const REJECT_HERALD = 2;
   const REJECT_EXTERNAL = 3;
   const REJECT_BROKEN = 4;
+  const REJECT_ENORMOUS = 5;
 
   protected $repositoryPHID;
   protected $epoch;

src/applications/repository/storage/PhabricatorRepositoryTransaction.php

@@ -16,6 +16,7 @@ final class PhabricatorRepositoryTransaction
   const TYPE_AUTOCLOSE = 'repo:autoclose';
   const TYPE_PUSH_POLICY = 'repo:push-policy';
   const TYPE_DANGEROUS = 'repo:dangerous';
+  const TYPE_ENORMOUS = 'repo:enormous';
   const TYPE_SLUG = 'repo:slug';
   const TYPE_SERVICE = 'repo:service';
   const TYPE_SYMBOLS_SOURCES = 'repo:symbol-source';
@@ -376,6 +377,16 @@ final class PhabricatorRepositoryTransaction
             '%s enabled protection against dangerous changes.',
             $this->renderHandleLink($author_phid));
         }
+      case self::TYPE_ENORMOUS:
+        if ($new) {
+          return pht(
+            '%s disabled protection against enormous changes.',
+            $this->renderHandleLink($author_phid));
+        } else {
+          return pht(
+            '%s enabled protection against enormous changes.',
+            $this->renderHandleLink($author_phid));
+        }
       case self::TYPE_SLUG:
         if (strlen($old) && !strlen($new)) {
           return pht(