Support (but do not actually enable) a maximum file size limit for Git repositories

Summary:
Depends on D19816. Ref T13216. See PHI908. See PHI750. In a few cases, users have pushed multi-gigabyte files full of various things that probably shouldn't be version controlled. This tends to create various headaches.

Add support for limiting the maximum size of any object. Specifically, we:

  - list all the objects each commit touches;
  - check their size after the commit applies;
  - if it's over the limit, reject the commit.

This change doesn't actually hook the limit up (the limit is always "0", i.e. unlimited), and doesn't have Mercurial or SVN support. The actual parser bit would probably be better in some other `Query/Parser` class eventually, too. But it at least roughly works.

Test Plan:
Changed the hard-coded limit to other values, tried to push stuff, got sensible results:

```
$ echo pew >> magic_missile.txt && git commit -am pew && git push
[master 98d07af] pew
 1 file changed, 1 insertion(+)
# Push received by "local.phacility.net", forwarding to cluster host.
# Acquiring write lock for repository "spellbook"...
# Acquired write lock immediately.
# Acquiring read lock for repository "spellbook" on device "local.phacility.net"...
# Acquired read lock immediately.
# Device "local.phacility.net" is already a cluster leader and does not need to be synchronized.
# Ready to receive on cluster host "local.phacility.net".
Counting objects: 49, done.
Delta compression using up to 8 threads.
Compressing objects: 100% (48/48), done.
Writing objects: 100% (49/49), 3.44 KiB | 1.72 MiB/s, done.
Total 49 (delta 30), reused 0 (delta 0)
remote: +---------------------------------------------------------------+
remote: |      * * * PUSH REJECTED BY EVIL DRAGON BUREAUCRATS * * *     |
remote: +---------------------------------------------------------------+
remote:              \
remote:               \                    ^    /^
remote:                \                  / \  // \
remote:                 \   |\___/|      /   \//  .\
remote:                  \  /V  V  \__  /    //  | \ \           *----*
remote:                    /     /  \/_/    //   |  \  \          \   |
remote:                    @___@`    \/_   //    |   \   \         \/\ \
remote:                   0/0/|       \/_ //     |    \    \         \  \
remote:               0/0/0/0/|        \///      |     \     \       |  |
remote:            0/0/0/0/0/_|_ /   (  //       |      \     _\     |  /
remote:         0/0/0/0/0/0/`/,_ _ _/  ) ; -.    |    _ _\.-~       /   /
remote:                     ,-}        _      *-.|.-~-.           .~    ~
remote:   *     \__/         `/\      /                 ~-. _ .-~      /
remote:    \____(Oo)            *.   }            {                   /
remote:    (    (..)           .----~-.\        \-`                 .~
remote:    //___\\  \ DENIED!  ///.----..<        \             _ -~
remote:   //     \\                ///-._ _ _ _ _ _ _{^ - - - - ~
remote:
remote:
remote: OVERSIZED FILE
remote: This repository ("spellbook") is configured with a maximum individual file size limit, but you are pushing a change ("98d07af863e799509e7c3a639404d216f9fc79c7") which causes the size of a file ("magic_missile.txt") to exceed the limit. The commit makes the file 317 bytes long, but the limit for this repository is 1 bytes.
remote:
# Released cluster write lock.
To ssh://local.phacility.com/source/spellbook.git
 ! [remote rejected] master -> master (pre-receive hook declined)
error: failed to push some refs to 'ssh://epriestley@local.phacility.com/source/spellbook.git'
```

Reviewers: amckinley

Reviewed By: amckinley

Subscribers: joshuaspence

Maniphest Tasks: T13216

Differential Revision: https://secure.phabricator.com/D19817

src/applications/diffusion/engine/DiffusionCommitHookEngine.php

@@ -164,6 +164,16 @@ final class DiffusionCommitHookEngine extends Phobject {
         $this->applyHeraldRefRules($ref_updates);
       }
 
+      try {
+        if (!$is_initial_import) {
+          $this->rejectOversizedFiles($content_updates);
+        }
+      } catch (DiffusionCommitHookRejectException $ex) {
+        // If we're rejecting oversized files, flag everything.
+        $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_OVERSIZED;
+        throw $ex;
+      }
+
       try {
         if (!$is_initial_import) {
           $this->rejectEnormousChanges($content_updates);
@@ -1255,6 +1265,139 @@ final class DiffusionCommitHookEngine extends Phobject {
     return $changesets;
   }
 
+  private function rejectOversizedFiles(array $content_updates) {
+    $repository = $this->getRepository();
+
+    // TODO: Allow repositories to be configured for a maximum filesize.
+    $limit = 0;
+
+    if (!$limit) {
+      return;
+    }
+
+    foreach ($content_updates as $update) {
+      $identifier = $update->getRefNew();
+
+      $sizes = $this->loadFileSizesForCommit($identifier);
+      foreach ($sizes as $path => $size) {
+        if ($size <= $limit) {
+          continue;
+        }
+
+        $message = pht(
+          'OVERSIZED FILE'.
+          "\n".
+          'This repository ("%s") is configured with a maximum individual '.
+          'file size limit, but you are pushing a change ("%s") which causes '.
+          'the size of a file ("%s") to exceed the limit. The commit makes '.
+          'the file %s bytes long, but the limit for this repository is '.
+          '%s bytes.',
+          $repository->getDisplayName(),
+          $identifier,
+          $path,
+          new PhutilNumber($size),
+          new PhutilNumber($limit));
+
+        throw new DiffusionCommitHookRejectException($message);
+      }
+    }
+  }
+
+  public function loadFileSizesForCommit($identifier) {
+    $repository = $this->getRepository();
+    $vcs = $repository->getVersionControlSystem();
+
+    $path_sizes = array();
+
+    switch ($vcs) {
+      case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
+        list($paths_raw) = $repository->execxLocalCommand(
+          'diff-tree -z -r --no-commit-id %s --',
+          $identifier);
+
+        // With "-z" we get "<fields>\0<filename>\0" for each line. Group the
+        // delimited text into "<fields>, <filename>" pairs.
+        $paths_raw = trim($paths_raw, "\0");
+        $paths_raw = explode("\0", $paths_raw);
+        if (count($paths_raw) % 2) {
+          throw new Exception(
+            pht(
+              'Unexpected number of output lines from "git diff-tree" when '.
+              'processing commit ("%s"): got %s lines, expected an even '.
+              'number.',
+              $identifier,
+              phutil_count($paths_raw)));
+        }
+        $paths_raw = array_chunk($paths_raw, 2);
+
+        $paths = array();
+        foreach ($paths_raw as $path_raw) {
+          list($fields, $pathname) = $path_raw;
+          $fields = explode(' ', $fields);
+
+          // Fields are:
+          //
+          //    :100644 100644 aaaa bbbb M
+          //
+          // [0] Old file mode.
+          // [1] New file mode.
+          // [2] Old object hash.
+          // [3] New object hash.
+          // [4] Change mode.
+
+          $paths[] = array(
+            'path' => $pathname,
+            'newHash' => $fields[3],
+          );
+        }
+
+        if ($paths) {
+          $check_paths = array();
+          foreach ($paths as $path) {
+            if ($path['newHash'] === self::EMPTY_HASH) {
+              $path_sizes[$path['path']] = 0;
+              continue;
+            }
+            $check_paths[$path['newHash']][] = $path['path'];
+          }
+
+          if ($check_paths) {
+            $future = $repository->getLocalCommandFuture(
+              'cat-file --batch-check=%s',
+              '%(objectsize)')
+              ->write(implode("\n", array_keys($check_paths)));
+
+            list($sizes) = $future->resolvex();
+            $sizes = trim($sizes);
+            $sizes = phutil_split_lines($sizes, false);
+            if (count($sizes) !== count($check_paths)) {
+              throw new Exception(
+                pht(
+                  'Unexpected number of output lines from "git cat-file" when '.
+                  'processing commit ("%s"): got %s lines, expected %s.',
+                  $identifier,
+                  phutil_count($sizes),
+                  phutil_count($check_paths)));
+            }
+
+            foreach ($check_paths as $object_hash => $path_names) {
+              $object_size = (int)array_shift($sizes);
+              foreach ($path_names as $path_name) {
+                $path_sizes[$path_name] = $object_size;
+              }
+            }
+          }
+        }
+        break;
+      default:
+        throw new Exception(
+          pht(
+            'File size limits are not supported for this VCS.'));
+    }
+
+    return $path_sizes;
+  }
+
   public function loadCommitRefForCommit($identifier) {
     $repository = $this->getRepository();
     $vcs = $repository->getVersionControlSystem();

src/applications/repository/storage/PhabricatorRepositoryPushLog.php

@@ -24,6 +24,7 @@ final class PhabricatorRepositoryPushLog
   const CHANGEFLAG_REWRITE = 8;
   const CHANGEFLAG_DANGEROUS = 16;
   const CHANGEFLAG_ENORMOUS = 32;
+  const CHANGEFLAG_OVERSIZED = 64;
 
   const REJECT_ACCEPT = 0;
   const REJECT_DANGEROUS = 1;
@@ -31,6 +32,7 @@ final class PhabricatorRepositoryPushLog
   const REJECT_EXTERNAL = 3;
   const REJECT_BROKEN = 4;
   const REJECT_ENORMOUS = 5;
+  const REJECT_OVERSIZED = 6;
 
   protected $repositoryPHID;
   protected $epoch;
@@ -63,6 +65,7 @@ final class PhabricatorRepositoryPushLog
       self::CHANGEFLAG_REWRITE => pht('Rewrite'),
       self::CHANGEFLAG_DANGEROUS => pht('Dangerous'),
       self::CHANGEFLAG_ENORMOUS => pht('Enormous'),
+      self::CHANGEFLAG_OVERSIZED => pht('Oversized'),
     );
   }
 
@@ -74,6 +77,7 @@ final class PhabricatorRepositoryPushLog
       self::REJECT_EXTERNAL => pht('Rejected: External Hook'),
       self::REJECT_BROKEN => pht('Rejected: Broken'),
       self::REJECT_ENORMOUS => pht('Rejected: Enormous'),
+      self::REJECT_OVERSIZED => pht('Rejected: Oversized File'),
     );
   }