From e08ba99dd3db49c073c37577d849c03b4d9ea24f Mon Sep 17 00:00:00 2001 From: epriestley Date: Mon, 8 Apr 2019 08:46:47 -0700 Subject: [PATCH] Proxy the "git upload-pack" wire protocol Summary: Depends on D20380. Ref T8093. When prototypes are enabled, inject a (hopefully?) no-op proxy into the Git wire protocol. This proxy decodes "git upload-pack" and allows the list of references to be rewritten, in a similar way to how we already proxy the Subversion protocol to rewrite URIs and proxy the Mercurial protocol to distinguish between read and write operations. The piece we care about comes at the beginning, and looks like this: ``` \0\n \n \n ... <0000> ``` We can add, remove, or modify this section to make it appear that the server has different refs than the refs that exist on disk. Things I have tried: - `git ls-remote` - `git ls-remote` where the server hides some refs. - `git fetch` where the fetch is a no-op. Things I have not tried: - `git fetch` where the fetch is not a no-op. - Tricking things into doing protocol v2. Or: I tried this, I wasn't successful. In v2, additional "\0" tricks are used to hide data in the capabilities, I think? - `git ls-remote` where we rewrite/hide the first ref in the list, and need to move the capabilities frame elsewhere. - `git ls-remote` where the server has no refs at all, or we remove every ref. So the "interesting" piece of this works, but it almost certainly needs some cleanup to survive interaction with the real world. Test Plan: See above. Reviewers: amckinley Reviewed By: amckinley Maniphest Tasks: T8093 Differential Revision: https://secure.phabricator.com/D20381 --- src/__phutil_library_map__.php | 4 + .../DiffusionGitUploadPackWireProtocol.php | 335 ++++++++++++++++++ .../protocol/DiffusionGitWireProtocol.php | 19 + .../diffusion/ssh/DiffusionGitSSHWorkflow.php | 26 +- .../ssh/DiffusionGitUploadPackSSHWorkflow.php | 10 + .../log/PhabricatorProtocolLog.php | 20 ++ 6 files changed, 412 insertions(+), 2 deletions(-) create mode 100644 src/applications/diffusion/protocol/DiffusionGitUploadPackWireProtocol.php create mode 100644 src/applications/diffusion/protocol/DiffusionGitWireProtocol.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 86c35f105e..b171d6b072 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -813,6 +813,8 @@ phutil_register_library_map(array( 'DiffusionGitResponse' => 'applications/diffusion/response/DiffusionGitResponse.php', 'DiffusionGitSSHWorkflow' => 'applications/diffusion/ssh/DiffusionGitSSHWorkflow.php', 'DiffusionGitUploadPackSSHWorkflow' => 'applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php', + 'DiffusionGitUploadPackWireProtocol' => 'applications/diffusion/protocol/DiffusionGitUploadPackWireProtocol.php', + 'DiffusionGitWireProtocol' => 'applications/diffusion/protocol/DiffusionGitWireProtocol.php', 'DiffusionGraphController' => 'applications/diffusion/controller/DiffusionGraphController.php', 'DiffusionHistoryController' => 'applications/diffusion/controller/DiffusionHistoryController.php', 'DiffusionHistoryListView' => 'applications/diffusion/view/DiffusionHistoryListView.php', @@ -6460,6 +6462,8 @@ phutil_register_library_map(array( 'DiffusionRepositoryClusterEngineLogInterface', ), 'DiffusionGitUploadPackSSHWorkflow' => 'DiffusionGitSSHWorkflow', + 'DiffusionGitUploadPackWireProtocol' => 'DiffusionGitWireProtocol', + 'DiffusionGitWireProtocol' => 'Phobject', 'DiffusionGraphController' => 'DiffusionController', 'DiffusionHistoryController' => 'DiffusionController', 'DiffusionHistoryListView' => 'DiffusionHistoryView', diff --git a/src/applications/diffusion/protocol/DiffusionGitUploadPackWireProtocol.php b/src/applications/diffusion/protocol/DiffusionGitUploadPackWireProtocol.php new file mode 100644 index 0000000000..5371586439 --- /dev/null +++ b/src/applications/diffusion/protocol/DiffusionGitUploadPackWireProtocol.php @@ -0,0 +1,335 @@ +readBuffer === null) { + $this->readBuffer = new PhutilRope(); + } + $buffer = $this->readBuffer; + + $buffer->append($bytes); + + while (true) { + $len = $buffer->getByteLength(); + switch ($this->readMode) { + case 'length': + // We're expecting 4 bytes containing the length of the protocol + // frame as hexadecimal in ASCII text, like "01ab". Wait until we + // see at least 4 bytes on the wire. + if ($len < 4) { + if ($len > 0) { + $bytes = $this->peekBytes($len); + if (!preg_match('/^[0-9a-f]+\z/', $bytes)) { + throw new Exception( + pht( + 'Bad frame length character in Git protocol ("%s"), '. + 'expected a 4-digit hexadecimal value encoded as ASCII '. + 'text.', + $bytes)); + } + } + + // We can't make any more progress until we get enough bytes, so + // we're done with state processing. + break 2; + } + + $frame_length = $this->readBytes(4); + $frame_length = hexdec($frame_length); + + // Note that the frame length includes the 4 header bytes, so we + // usually expect a length of 5 or larger. Frames with length 0 + // are boundaries. + if ($frame_length === 0) { + $this->readFrames[] = $this->newProtocolFrame('null', ''); + } else if ($frame_length >= 1 && $frame_length <= 3) { + throw new Exception( + pht( + 'Encountered Git protocol frame with unexpected frame '. + 'length (%s)!', + $frame_length)); + } else { + $this->readFrameLength = $frame_length - 4; + $this->readMode = 'frame'; + } + + break; + case 'frame': + // We're expecting a protocol frame of a specified length. Note that + // it is possible for a frame to have length 0. + + // We don't have enough bytes yet, so wait for more. + if ($len < $this->readFrameLength) { + break 2; + } + + if ($this->readFrameLength > 0) { + $bytes = $this->readBytes($this->readFrameLength); + } else { + $bytes = ''; + } + + // Emit a protocol frame. + $this->readFrames[] = $this->newProtocolFrame('data', $bytes); + $this->readMode = 'length'; + break; + } + } + + while (true) { + switch ($this->readFrameMode) { + case 'refs': + if (!$this->readFrames) { + break 2; + } + + foreach ($this->readFrames as $key => $frame) { + unset($this->readFrames[$key]); + + if ($frame['type'] === 'null') { + $ref_frames = $this->refFrames; + $this->refFrames = array(); + + $ref_frames[] = $frame; + + $this->readMessages[] = $this->newProtocolRefMessage($ref_frames); + $this->readFrameMode = 'passthru'; + break; + } else { + $this->refFrames[] = $frame; + } + } + + break; + case 'passthru': + if (!$this->readFrames) { + break 2; + } + + $this->readMessages[] = $this->newProtocolDataMessage( + $this->readFrames); + $this->readFrames = array(); + + break; + } + } + + $wire = array(); + foreach ($this->readMessages as $key => $message) { + $wire[] = $message; + unset($this->readMessages[$key]); + } + $wire = implode('', $wire); + + return $wire; + } + + public function willWriteBytes($bytes) { + return $bytes; + } + + private function readBytes($count) { + $buffer = $this->readBuffer; + + $bytes = $buffer->getPrefixBytes($count); + $buffer->removeBytesFromHead($count); + + return $bytes; + } + + private function peekBytes($count) { + $buffer = $this->readBuffer; + return $buffer->getPrefixBytes($count); + } + + private function newProtocolFrame($type, $bytes) { + return array( + 'type' => $type, + 'length' => strlen($bytes), + 'bytes' => $bytes, + ); + } + + private function newProtocolRefMessage(array $frames) { + $head_key = head_key($frames); + $last_key = last_key($frames); + + $output = array(); + foreach ($frames as $key => $frame) { + $is_last = ($key === $last_key); + if ($is_last) { + $output[] = $frame; + // This is a "0000" frame at the end of the list of refs, so we pass + // it through unmodified. + continue; + } + + $is_first = ($key === $head_key); + + // Otherwise, we expect a list of: + // + // \0 + // + // ... + + $bytes = $frame['bytes']; + $matches = array(); + if ($is_first) { + $ok = preg_match( + '('. + '^'. + '(?P[0-9a-f]{40})'. + ' '. + '(?P[^\0\n]+)'. + '\0'. + '(?P[^\n]+)'. + '\n'. + '\z'. + ')', + $bytes, + $matches); + if (!$ok) { + throw new Exception( + pht( + 'Unexpected "git upload-pack" initial protocol frame: expected '. + '" \0\n", got "%s".', + $bytes)); + } + } else { + $ok = preg_match( + '('. + '^'. + '(?P[0-9a-f]{40})'. + ' '. + '(?P[^\0\n]+)'. + '\n'. + '\z'. + ')', + $bytes, + $matches); + if (!$ok) { + throw new Exception( + pht( + 'Unexpected "git upload-pack" protocol frame: expected '. + '" \n", got "%s".', + $bytes)); + } + } + + $hash = $matches['hash']; + $name = $matches['name']; + $capabilities = idx($matches, 'capabilities'); + + $ref = array( + 'hash' => $hash, + 'name' => $name, + 'capabilities' => $capabilities, + ); + + $old_ref = $ref; + + $ref = $this->willReadRef($ref); + + $new_ref = $ref; + + $this->didRewriteRef($old_ref, $new_ref); + + if ($ref === null) { + continue; + } + + if (isset($ref['capabilities'])) { + $result = sprintf( + "%s %s\0%s\n", + $ref['hash'], + $ref['name'], + $ref['capabilities']); + } else { + $result = sprintf( + "%s %s\n", + $ref['hash'], + $ref['name']); + } + + $output[] = $this->newProtocolFrame('data', $result); + } + + return $this->newProtocolDataMessage($output); + } + + private function newProtocolDataMessage(array $frames) { + $message = array(); + + foreach ($frames as $frame) { + switch ($frame['type']) { + case 'null': + $message[] = '0000'; + break; + case 'data': + $message[] = sprintf( + '%04x%s', + $frame['length'] + 4, + $frame['bytes']); + break; + } + } + + $message = implode('', $message); + + return $message; + } + + private function willReadRef(array $ref) { + return $ref; + } + + private function didRewriteRef($old_ref, $new_ref) { + $log = $this->getProtocolLog(); + if (!$log) { + return; + } + + if (!$old_ref) { + $old_name = null; + } else { + $old_name = $old_ref['name']; + } + + if (!$new_ref) { + $new_name = null; + } else { + $new_name = $new_ref['name']; + } + + if ($old_name === $new_name) { + return; + } + + if ($old_name === null) { + $old_name = ''; + } + + if ($new_name === null) { + $new_name = ''; + } + + $log->didWriteFrame( + pht( + 'Rewrite Ref: %s -> %s', + $old_name, + $new_name)); + } + +} diff --git a/src/applications/diffusion/protocol/DiffusionGitWireProtocol.php b/src/applications/diffusion/protocol/DiffusionGitWireProtocol.php new file mode 100644 index 0000000000..9ea8d96cf4 --- /dev/null +++ b/src/applications/diffusion/protocol/DiffusionGitWireProtocol.php @@ -0,0 +1,19 @@ +protocolLog = $protocol_log; + return $this; + } + + final public function getProtocolLog() { + return $this->protocolLog; + } + + abstract public function willReadBytes($bytes); + abstract public function willWriteBytes($bytes); + +} diff --git a/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php index 578c6b1e03..d9cc8063d5 100644 --- a/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php +++ b/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php @@ -7,6 +7,8 @@ abstract class DiffusionGitSSHWorkflow private $engineLogProperties = array(); private $protocolLog; + private $wireProtocol; + protected function writeError($message) { // Git assumes we'll add our own newlines. return parent::writeError($message."\n"); @@ -74,14 +76,24 @@ abstract class DiffusionGitSSHWorkflow return null; } - protected function getProtocolLog() { + final protected function getProtocolLog() { return $this->protocolLog; } - protected function setProtocolLog(PhabricatorProtocolLog $log) { + final protected function setProtocolLog(PhabricatorProtocolLog $log) { $this->protocolLog = $log; } + final protected function getWireProtocol() { + return $this->wireProtocol; + } + + final protected function setWireProtocol( + DiffusionGitWireProtocol $protocol) { + $this->wireProtocol = $protocol; + return $this; + } + public function willWriteMessageCallback( PhabricatorSSHPassthruCommand $command, $message) { @@ -91,6 +103,11 @@ abstract class DiffusionGitSSHWorkflow $log->didWriteBytes($message); } + $protocol = $this->getWireProtocol(); + if ($protocol) { + $message = $protocol->willWriteBytes($message); + } + return $message; } @@ -103,6 +120,11 @@ abstract class DiffusionGitSSHWorkflow $log->didReadBytes($message); } + $protocol = $this->getWireProtocol(); + if ($protocol) { + $message = $protocol->willReadBytes($message); + } + return $message; } diff --git a/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php index c8c779feb3..7e1f4a4f33 100644 --- a/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php +++ b/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php @@ -60,6 +60,16 @@ final class DiffusionGitUploadPackSSHWorkflow extends DiffusionGitSSHWorkflow { $log->didStartSession($command); } + if (!$is_proxy) { + if (PhabricatorEnv::getEnvConfig('phabricator.show-prototypes')) { + $protocol = new DiffusionGitUploadPackWireProtocol(); + if ($log) { + $protocol->setProtocolLog($log); + } + $this->setWireProtocol($protocol); + } + } + $err = $this->newPassthruCommand() ->setIOChannel($this->getIOChannel()) ->setCommandChannelFromExecFuture($future) diff --git a/src/infrastructure/log/PhabricatorProtocolLog.php b/src/infrastructure/log/PhabricatorProtocolLog.php index d2775b27de..089d35a229 100644 --- a/src/infrastructure/log/PhabricatorProtocolLog.php +++ b/src/infrastructure/log/PhabricatorProtocolLog.php @@ -41,6 +41,26 @@ final class PhabricatorProtocolLog $this->buffer[] = $bytes; } + public function didReadFrame($frame) { + $this->writeFrame('<*', $frame); + } + + public function didWriteFrame($frame) { + $this->writeFrame('>*', $frame); + } + + private function writeFrame($header, $frame) { + $this->flush(); + + $frame = explode("\n", $frame); + foreach ($frame as $key => $line) { + $frame[$key] = $header.' '.$this->escapeBytes($line); + } + $frame = implode("\n", $frame)."\n\n"; + + $this->writeMessage($frame); + } + private function setMode($mode) { if ($this->mode === $mode) { return $this;