Proxy the "git upload-pack" wire protocol

Summary:
Depends on D20380. Ref T8093. When prototypes are enabled, inject a (hopefully?) no-op proxy into the Git wire protocol.

This proxy decodes "git upload-pack" and allows the list of references to be rewritten, in a similar way to how we already proxy the Subversion protocol to rewrite URIs and proxy the Mercurial protocol to distinguish between read and write operations.

The piece we care about comes at the beginning, and looks like this:

```
<frame-length><ref-hash> <ref-name>\0<server-capabilities>\n
<frame-length><ref-hash> <ref-name>\n
<frame-length><ref-hash> <ref-name>\n
...
<0000>
```

We can add, remove, or modify this section to make it appear that the server has different refs than the refs that exist on disk.

Things I have tried:

  - `git ls-remote`
  - `git ls-remote` where the server hides some refs.
  - `git fetch` where the fetch is a no-op.

Things I have not tried:

  - `git fetch` where the fetch is not a no-op.
  - Tricking things into doing protocol v2. Or: I tried this, I wasn't successful. In v2, additional "\0" tricks are used to hide data in the capabilities, I think?
  - `git ls-remote` where we rewrite/hide the first ref in the list, and need to move the capabilities frame elsewhere.
  - `git ls-remote` where the server has no refs at all, or we remove every ref.

So the "interesting" piece of this works, but it almost certainly needs some cleanup to survive interaction with the real world.

Test Plan: See above.

Reviewers: amckinley

Reviewed By: amckinley

Maniphest Tasks: T8093

Differential Revision: https://secure.phabricator.com/D20381

src/__phutil_library_map__.php

@@ -813,6 +813,8 @@ phutil_register_library_map(array(
     'DiffusionGitResponse' => 'applications/diffusion/response/DiffusionGitResponse.php',
     'DiffusionGitSSHWorkflow' => 'applications/diffusion/ssh/DiffusionGitSSHWorkflow.php',
     'DiffusionGitUploadPackSSHWorkflow' => 'applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php',
+    'DiffusionGitUploadPackWireProtocol' => 'applications/diffusion/protocol/DiffusionGitUploadPackWireProtocol.php',
+    'DiffusionGitWireProtocol' => 'applications/diffusion/protocol/DiffusionGitWireProtocol.php',
     'DiffusionGraphController' => 'applications/diffusion/controller/DiffusionGraphController.php',
     'DiffusionHistoryController' => 'applications/diffusion/controller/DiffusionHistoryController.php',
     'DiffusionHistoryListView' => 'applications/diffusion/view/DiffusionHistoryListView.php',
@@ -6460,6 +6462,8 @@ phutil_register_library_map(array(
       'DiffusionRepositoryClusterEngineLogInterface',
     ),
     'DiffusionGitUploadPackSSHWorkflow' => 'DiffusionGitSSHWorkflow',
+    'DiffusionGitUploadPackWireProtocol' => 'DiffusionGitWireProtocol',
+    'DiffusionGitWireProtocol' => 'Phobject',
     'DiffusionGraphController' => 'DiffusionController',
     'DiffusionHistoryController' => 'DiffusionController',
     'DiffusionHistoryListView' => 'DiffusionHistoryView',

src/applications/diffusion/protocol/DiffusionGitUploadPackWireProtocol.php

@@ -0,0 +1,335 @@
+<?php
+
+final class DiffusionGitUploadPackWireProtocol
+  extends DiffusionGitWireProtocol {
+
+  private $readMode = 'length';
+  private $readBuffer;
+  private $readFrameLength;
+  private $readFrames = array();
+
+  private $readFrameMode = 'refs';
+  private $refFrames = array();
+
+  private $readMessages = array();
+
+  public function willReadBytes($bytes) {
+    if ($this->readBuffer === null) {
+      $this->readBuffer = new PhutilRope();
+    }
+    $buffer = $this->readBuffer;
+
+    $buffer->append($bytes);
+
+    while (true) {
+      $len = $buffer->getByteLength();
+      switch ($this->readMode) {
+        case 'length':
+          // We're expecting 4 bytes containing the length of the protocol
+          // frame as hexadecimal in ASCII text, like "01ab". Wait until we
+          // see at least 4 bytes on the wire.
+          if ($len < 4) {
+            if ($len > 0) {
+              $bytes = $this->peekBytes($len);
+              if (!preg_match('/^[0-9a-f]+\z/', $bytes)) {
+                throw new Exception(
+                  pht(
+                    'Bad frame length character in Git protocol ("%s"), '.
+                    'expected a 4-digit hexadecimal value encoded as ASCII '.
+                    'text.',
+                    $bytes));
+              }
+            }
+
+            // We can't make any more progress until we get enough bytes, so
+            // we're done with state processing.
+            break 2;
+          }
+
+          $frame_length = $this->readBytes(4);
+          $frame_length = hexdec($frame_length);
+
+          // Note that the frame length includes the 4 header bytes, so we
+          // usually expect a length of 5 or larger. Frames with length 0
+          // are boundaries.
+          if ($frame_length === 0) {
+            $this->readFrames[] = $this->newProtocolFrame('null', '');
+          } else if ($frame_length >= 1 && $frame_length <= 3) {
+            throw new Exception(
+              pht(
+                'Encountered Git protocol frame with unexpected frame '.
+                'length (%s)!',
+                $frame_length));
+          } else {
+            $this->readFrameLength = $frame_length - 4;
+            $this->readMode = 'frame';
+          }
+
+          break;
+        case 'frame':
+          // We're expecting a protocol frame of a specified length. Note that
+          // it is possible for a frame to have length 0.
+
+          // We don't have enough bytes yet, so wait for more.
+          if ($len < $this->readFrameLength) {
+            break 2;
+          }
+
+          if ($this->readFrameLength > 0) {
+            $bytes = $this->readBytes($this->readFrameLength);
+          } else {
+            $bytes = '';
+          }
+
+          // Emit a protocol frame.
+          $this->readFrames[] = $this->newProtocolFrame('data', $bytes);
+          $this->readMode = 'length';
+          break;
+      }
+    }
+
+    while (true) {
+      switch ($this->readFrameMode) {
+        case 'refs':
+          if (!$this->readFrames) {
+            break 2;
+          }
+
+          foreach ($this->readFrames as $key => $frame) {
+            unset($this->readFrames[$key]);
+
+            if ($frame['type'] === 'null') {
+              $ref_frames = $this->refFrames;
+              $this->refFrames = array();
+
+              $ref_frames[] = $frame;
+
+              $this->readMessages[] = $this->newProtocolRefMessage($ref_frames);
+              $this->readFrameMode = 'passthru';
+              break;
+            } else {
+              $this->refFrames[] = $frame;
+            }
+          }
+
+          break;
+        case 'passthru':
+          if (!$this->readFrames) {
+            break 2;
+          }
+
+          $this->readMessages[] = $this->newProtocolDataMessage(
+            $this->readFrames);
+          $this->readFrames = array();
+
+          break;
+      }
+    }
+
+    $wire = array();
+    foreach ($this->readMessages as $key => $message) {
+      $wire[] = $message;
+      unset($this->readMessages[$key]);
+    }
+    $wire = implode('', $wire);
+
+    return $wire;
+  }
+
+  public function willWriteBytes($bytes) {
+    return $bytes;
+  }
+
+  private function readBytes($count) {
+    $buffer = $this->readBuffer;
+
+    $bytes = $buffer->getPrefixBytes($count);
+    $buffer->removeBytesFromHead($count);
+
+    return $bytes;
+  }
+
+  private function peekBytes($count) {
+    $buffer = $this->readBuffer;
+    return $buffer->getPrefixBytes($count);
+  }
+
+  private function newProtocolFrame($type, $bytes) {
+    return array(
+      'type' => $type,
+      'length' => strlen($bytes),
+      'bytes' => $bytes,
+    );
+  }
+
+  private function newProtocolRefMessage(array $frames) {
+    $head_key = head_key($frames);
+    $last_key = last_key($frames);
+
+    $output = array();
+    foreach ($frames as $key => $frame) {
+      $is_last = ($key === $last_key);
+      if ($is_last) {
+        $output[] = $frame;
+        // This is a "0000" frame at the end of the list of refs, so we pass
+        // it through unmodified.
+        continue;
+      }
+
+      $is_first = ($key === $head_key);
+
+      // Otherwise, we expect a list of:
+      //
+      //   <hash> <ref-name>\0<capabilities>
+      //   <hash> <ref-name>
+      //   ...
+
+      $bytes = $frame['bytes'];
+      $matches = array();
+      if ($is_first) {
+        $ok = preg_match(
+          '('.
+            '^'.
+            '(?P<hash>[0-9a-f]{40})'.
+            ' '.
+            '(?P<name>[^\0\n]+)'.
+            '\0'.
+            '(?P<capabilities>[^\n]+)'.
+            '\n'.
+            '\z'.
+          ')',
+          $bytes,
+          $matches);
+        if (!$ok) {
+          throw new Exception(
+            pht(
+              'Unexpected "git upload-pack" initial protocol frame: expected '.
+              '"<hash> <name>\0<capabilities>\n", got "%s".',
+              $bytes));
+        }
+      } else {
+        $ok = preg_match(
+          '('.
+            '^'.
+            '(?P<hash>[0-9a-f]{40})'.
+            ' '.
+            '(?P<name>[^\0\n]+)'.
+            '\n'.
+            '\z'.
+          ')',
+          $bytes,
+          $matches);
+        if (!$ok) {
+          throw new Exception(
+            pht(
+              'Unexpected "git upload-pack" protocol frame: expected '.
+              '"<hash> <name>\n", got "%s".',
+              $bytes));
+        }
+      }
+
+      $hash = $matches['hash'];
+      $name = $matches['name'];
+      $capabilities = idx($matches, 'capabilities');
+
+      $ref = array(
+        'hash' => $hash,
+        'name' => $name,
+        'capabilities' => $capabilities,
+      );
+
+      $old_ref = $ref;
+
+      $ref = $this->willReadRef($ref);
+
+      $new_ref = $ref;
+
+      $this->didRewriteRef($old_ref, $new_ref);
+
+      if ($ref === null) {
+        continue;
+      }
+
+      if (isset($ref['capabilities'])) {
+        $result = sprintf(
+          "%s %s\0%s\n",
+          $ref['hash'],
+          $ref['name'],
+          $ref['capabilities']);
+      } else {
+        $result = sprintf(
+          "%s %s\n",
+          $ref['hash'],
+          $ref['name']);
+      }
+
+      $output[] = $this->newProtocolFrame('data', $result);
+    }
+
+    return $this->newProtocolDataMessage($output);
+  }
+
+  private function newProtocolDataMessage(array $frames) {
+    $message = array();
+
+    foreach ($frames as $frame) {
+      switch ($frame['type']) {
+        case 'null':
+          $message[] = '0000';
+          break;
+        case 'data':
+          $message[] = sprintf(
+            '%04x%s',
+            $frame['length'] + 4,
+            $frame['bytes']);
+          break;
+      }
+    }
+
+    $message = implode('', $message);
+
+    return $message;
+  }
+
+  private function willReadRef(array $ref) {
+    return $ref;
+  }
+
+  private function didRewriteRef($old_ref, $new_ref) {
+    $log = $this->getProtocolLog();
+    if (!$log) {
+      return;
+    }
+
+    if (!$old_ref) {
+      $old_name = null;
+    } else {
+      $old_name = $old_ref['name'];
+    }
+
+    if (!$new_ref) {
+      $new_name = null;
+    } else {
+      $new_name = $new_ref['name'];
+    }
+
+    if ($old_name === $new_name) {
+      return;
+    }
+
+    if ($old_name === null) {
+      $old_name = '<null>';
+    }
+
+    if ($new_name === null) {
+      $new_name = '<null>';
+    }
+
+    $log->didWriteFrame(
+      pht(
+        'Rewrite Ref: %s -> %s',
+        $old_name,
+        $new_name));
+  }
+
+}

src/applications/diffusion/protocol/DiffusionGitWireProtocol.php

@@ -0,0 +1,19 @@
+<?php
+
+abstract class DiffusionGitWireProtocol extends Phobject {
+
+  private $protocolLog;
+
+  final public function setProtocolLog(PhabricatorProtocolLog $protocol_log) {
+    $this->protocolLog = $protocol_log;
+    return $this;
+  }
+
+  final public function getProtocolLog() {
+    return $this->protocolLog;
+  }
+
+  abstract public function willReadBytes($bytes);
+  abstract public function willWriteBytes($bytes);
+
+}

src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php

@@ -7,6 +7,8 @@ abstract class DiffusionGitSSHWorkflow
   private $engineLogProperties = array();
   private $protocolLog;
 
+  private $wireProtocol;
+
   protected function writeError($message) {
     // Git assumes we'll add our own newlines.
     return parent::writeError($message."\n");
@@ -74,14 +76,24 @@ abstract class DiffusionGitSSHWorkflow
     return null;
   }
 
-  protected function getProtocolLog() {
+  final protected function getProtocolLog() {
     return $this->protocolLog;
   }
 
-  protected function setProtocolLog(PhabricatorProtocolLog $log) {
+  final protected function setProtocolLog(PhabricatorProtocolLog $log) {
     $this->protocolLog = $log;
   }
 
+  final protected function getWireProtocol() {
+    return $this->wireProtocol;
+  }
+
+  final protected function setWireProtocol(
+    DiffusionGitWireProtocol $protocol) {
+    $this->wireProtocol = $protocol;
+    return $this;
+  }
+
   public function willWriteMessageCallback(
     PhabricatorSSHPassthruCommand $command,
     $message) {
@@ -91,6 +103,11 @@ abstract class DiffusionGitSSHWorkflow
       $log->didWriteBytes($message);
     }
 
+    $protocol = $this->getWireProtocol();
+    if ($protocol) {
+      $message = $protocol->willWriteBytes($message);
+    }
+
     return $message;
   }
 
@@ -103,6 +120,11 @@ abstract class DiffusionGitSSHWorkflow
       $log->didReadBytes($message);
     }
 
+    $protocol = $this->getWireProtocol();
+    if ($protocol) {
+      $message = $protocol->willReadBytes($message);
+    }
+
     return $message;
   }
 

src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php

@@ -60,6 +60,16 @@ final class DiffusionGitUploadPackSSHWorkflow extends DiffusionGitSSHWorkflow {
       $log->didStartSession($command);
     }
 
+    if (!$is_proxy) {
+      if (PhabricatorEnv::getEnvConfig('phabricator.show-prototypes')) {
+        $protocol = new DiffusionGitUploadPackWireProtocol();
+        if ($log) {
+          $protocol->setProtocolLog($log);
+        }
+        $this->setWireProtocol($protocol);
+      }
+    }
+
     $err = $this->newPassthruCommand()
       ->setIOChannel($this->getIOChannel())
       ->setCommandChannelFromExecFuture($future)

src/infrastructure/log/PhabricatorProtocolLog.php

@@ -41,6 +41,26 @@ final class PhabricatorProtocolLog
     $this->buffer[] = $bytes;
   }
 
+  public function didReadFrame($frame) {
+    $this->writeFrame('<*', $frame);
+  }
+
+  public function didWriteFrame($frame) {
+    $this->writeFrame('>*', $frame);
+  }
+
+  private function writeFrame($header, $frame) {
+    $this->flush();
+
+    $frame = explode("\n", $frame);
+    foreach ($frame as $key => $line) {
+      $frame[$key] = $header.'  '.$this->escapeBytes($line);
+    }
+    $frame = implode("\n", $frame)."\n\n";
+
+    $this->writeMessage($frame);
+  }
+
   private function setMode($mode) {
     if ($this->mode === $mode) {
       return $this;