Implement a Git LFS link table and basic batch API

Summary:
Ref T7789. This implements:

  - A new table to store the `<objectHash, filePHID>` relationship between Git LFS files and Phabricator file objects.
  - A basic response to `batch` commands, which return actions for a list of files.

Test Plan:
Ran `git lfs push origin master`, got a little further than previously:

```
epriestley@orbital ~/dev/scratch/poemslocal $ git lfs push origin master
Git LFS: (2 of 1 files) 174.24 KB / 87.12 KB
Git LFS operation "upload/b7e0aeb82a03d627c6aa5fc1bbfd454b6789d9d9affc8607d40168fa18cf6c69" is not supported by this server.
Git LFS operation "upload/b7e0aeb82a03d627c6aa5fc1bbfd454b6789d9d9affc8607d40168fa18cf6c69" is not supported by this server.
```

With `GIT_TRACE=1`, this shows the batch part of the API going through.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T7789

Differential Revision: https://secure.phabricator.com/D15489

resources/sql/autopatches/20160317.lfs.01.ref.sql

@@ -0,0 +1,11 @@
+CREATE TABLE {$NAMESPACE}_repository.repository_gitlfsref (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  repositoryPHID VARBINARY(64) NOT NULL,
+  objectHash BINARY(64) NOT NULL,
+  byteSize BIGINT UNSIGNED NOT NULL,
+  authorPHID VARBINARY(64) NOT NULL,
+  filePHID VARBINARY(64) NOT NULL,
+  dateCreated INT UNSIGNED NOT NULL,
+  dateModified INT UNSIGNED NOT NULL,
+  UNIQUE KEY `key_hash` (repositoryPHID, objectHash)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

src/__phutil_library_map__.php

@@ -3100,6 +3100,8 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryEngine' => 'applications/repository/engine/PhabricatorRepositoryEngine.php',
     'PhabricatorRepositoryGitCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/PhabricatorRepositoryGitCommitChangeParserWorker.php',
     'PhabricatorRepositoryGitCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/PhabricatorRepositoryGitCommitMessageParserWorker.php',
+    'PhabricatorRepositoryGitLFSRef' => 'applications/repository/storage/PhabricatorRepositoryGitLFSRef.php',
+    'PhabricatorRepositoryGitLFSRefQuery' => 'applications/repository/query/PhabricatorRepositoryGitLFSRefQuery.php',
     'PhabricatorRepositoryGraphCache' => 'applications/repository/graphcache/PhabricatorRepositoryGraphCache.php',
     'PhabricatorRepositoryGraphStream' => 'applications/repository/daemon/PhabricatorRepositoryGraphStream.php',
     'PhabricatorRepositoryManagementCacheWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementCacheWorkflow.php',
@@ -7665,6 +7667,11 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryEngine' => 'Phobject',
     'PhabricatorRepositoryGitCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
     'PhabricatorRepositoryGitCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
+    'PhabricatorRepositoryGitLFSRef' => array(
+      'PhabricatorRepositoryDAO',
+      'PhabricatorPolicyInterface',
+    ),
+    'PhabricatorRepositoryGitLFSRefQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'PhabricatorRepositoryGraphCache' => 'Phobject',
     'PhabricatorRepositoryGraphStream' => 'Phobject',
     'PhabricatorRepositoryManagementCacheWorkflow' => 'PhabricatorRepositoryManagementWorkflow',

src/applications/diffusion/controller/DiffusionServeController.php

@@ -9,6 +9,8 @@ final class DiffusionServeController extends DiffusionController {
   private $gitLFSToken;
 
   public function setServiceViewer(PhabricatorUser $viewer) {
+    $this->getRequest()->setUser($viewer);
+
     $this->serviceViewer = $viewer;
     return $this;
   }
@@ -42,6 +44,7 @@ final class DiffusionServeController extends DiffusionController {
 
     $content_type = $request->getHTTPHeader('Content-Type');
     $user_agent = idx($_SERVER, 'HTTP_USER_AGENT');
+    $request_type = $request->getHTTPHeader('X-Phabricator-Request-Type');
 
     // This may have a "charset" suffix, so only match the prefix.
     $lfs_pattern = '(^application/vnd\\.git-lfs\\+json(;|\z))';
@@ -64,6 +67,10 @@ final class DiffusionServeController extends DiffusionController {
       // This is a Git LFS HTTP API request.
       $vcs = PhabricatorRepositoryType::REPOSITORY_TYPE_GIT;
       $this->isGitLFSRequest = true;
+    } else if ($request_type == 'git-lfs') {
+      // This is a Git LFS object content request.
+      $vcs = PhabricatorRepositoryType::REPOSITORY_TYPE_GIT;
+      $this->isGitLFSRequest = true;
     } else if ($request->getExists('cmd')) {
       // Mercurial also sends an Accept header like
       // "application/mercurial-0.1", and a User-Agent like
@@ -884,12 +891,140 @@ final class DiffusionServeController extends DiffusionController {
     }
 
     $path = $this->getGitLFSRequestPath($repository);
+    if ($path == 'objects/batch') {
+      return $this->serveGitLFSBatchRequest($repository, $viewer);
+    } else {
+      return DiffusionGitLFSResponse::newErrorResponse(
+        404,
+        pht(
+          'Git LFS operation "%s" is not supported by this server.',
+          $path));
+    }
+  }
 
-    return DiffusionGitLFSResponse::newErrorResponse(
-      404,
-      pht(
-        'Git LFS operation "%s" is not supported by this server.',
-        $path));
+  private function serveGitLFSBatchRequest(
+    PhabricatorRepository $repository,
+    PhabricatorUser $viewer) {
+
+    $input = PhabricatorStartup::getRawInput();
+    $input = phutil_json_decode($input);
+
+    $operation = idx($input, 'operation');
+    switch ($operation) {
+      case 'upload':
+        $want_upload = true;
+        break;
+      case 'download':
+        $want_upload = false;
+        break;
+      default:
+        return DiffusionGitLFSResponse::newErrorResponse(
+          404,
+          pht(
+            'Git LFS batch operation "%s" is not supported by this server.',
+            $operation));
+    }
+
+    $objects = idx($input, 'objects', array());
+
+    $hashes = array();
+    foreach ($objects as $object) {
+      $hashes[] = idx($object, 'oid');
+    }
+
+    if ($hashes) {
+      $refs = id(new PhabricatorRepositoryGitLFSRefQuery())
+        ->setViewer($viewer)
+        ->withRepositoryPHIDs(array($repository->getPHID()))
+        ->withObjectHashes($hashes)
+        ->execute();
+      $refs = mpull($refs, null, 'getObjectHash');
+    } else {
+      $refs = array();
+    }
+
+    $file_phids = mpull($refs, 'getFilePHID');
+    if ($file_phids) {
+      $files = id(new PhabricatorFileQuery())
+        ->setViewer($viewer)
+        ->withPHIDs(array($file_phids))
+        ->execute();
+      $files = mpull($files, null, 'getPHID');
+    } else {
+      $files = array();
+    }
+
+    $authorization = null;
+    $output = array();
+    foreach ($objects as $object) {
+      $oid = idx($object, 'oid');
+      $size = idx($object, 'size');
+      $ref = idx($refs, $oid);
+
+      // NOTE: If we already have a ref for this object, we only emit a
+      // "download" action. The client should not upload the file again.
+
+      $actions = array();
+      if ($ref) {
+        $file = idx($files, $ref->getFilePHID());
+        if ($file) {
+          $get_uri = $file->getCDNURIWithToken();
+          $actions['download'] = array(
+            'href' => $get_uri,
+          );
+        }
+      } else if ($want_upload) {
+        if (!$authorization) {
+          // Here, we could reuse the existing authorization if we have one,
+          // but it's a little simpler to just generate a new one
+          // unconditionally.
+          $authorization = $this->newGitLFSHTTPAuthorization(
+            $repository,
+            $viewer,
+            $operation);
+        }
+
+        $put_uri = $repository->getGitLFSURI("info/lfs/upload/{$oid}");
+
+        $actions['upload'] = array(
+          'href' => $put_uri,
+          'header' => array(
+            'Authorization' => $authorization,
+            'X-Phabricator-Request-Type' => 'git-lfs',
+          ),
+        );
+      }
+
+      $output[] = array(
+        'oid' => $oid,
+        'size' => $size,
+        'actions' => $actions,
+      );
+    }
+
+    $output = array(
+      'objects' => $output,
+    );
+
+    return id(new DiffusionGitLFSResponse())
+      ->setContent($output);
+  }
+
+  private function newGitLFSHTTPAuthorization(
+    PhabricatorRepository $repository,
+    PhabricatorUser $viewer,
+    $operation) {
+
+    $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
+
+    $authorization = DiffusionGitLFSTemporaryTokenType::newHTTPAuthorization(
+      $repository,
+      $viewer,
+      $operation);
+
+    unset($unguarded);
+
+    return $authorization;
   }
 
   private function getGitLFSRequestPath(PhabricatorRepository $repository) {

src/applications/diffusion/gitlfs/DiffusionGitLFSAuthenticateWorkflow.php

@@ -83,25 +83,15 @@ final class DiffusionGitLFSAuthenticateWorkflow
     // on this host, and does not require the user to have a VCS password.
 
     $user = $this->getUser();
-    $headers = array();
 
-    $lfs_user = DiffusionGitLFSTemporaryTokenType::HTTP_USERNAME;
-    $lfs_pass = Filesystem::readRandomCharacters(32);
-    $lfs_hash = PhabricatorHash::digest($lfs_pass);
+    $authorization = DiffusionGitLFSTemporaryTokenType::newHTTPAuthorization(
+      $repository,
+      $user,
+      $operation);
 
-    $ttl = PhabricatorTime::getNow() + phutil_units('1 day in seconds');
-
-    $token = id(new PhabricatorAuthTemporaryToken())
-      ->setTokenResource($repository->getPHID())
-      ->setTokenType(DiffusionGitLFSTemporaryTokenType::TOKENTYPE)
-      ->setTokenCode($lfs_hash)
-      ->setUserPHID($user->getPHID())
-      ->setTemporaryTokenProperty('lfs.operation', $operation)
-      ->setTokenExpires($ttl)
-      ->save();
-
-    $authorization_header = base64_encode($lfs_user.':'.$lfs_pass);
-    $headers['Authorization'] = 'Basic '.$authorization_header;
+    $headers = array(
+      'authorization' => $authorization,
+    );
 
     $result = array(
       'header' => $headers,

src/applications/diffusion/gitlfs/DiffusionGitLFSTemporaryTokenType.php

@@ -15,4 +15,28 @@ final class DiffusionGitLFSTemporaryTokenType
     return pht('Git LFS Token');
   }
 
+  public static function newHTTPAuthorization(
+    PhabricatorRepository $repository,
+    PhabricatorUser $viewer,
+    $operation) {
+
+    $lfs_user = self::HTTP_USERNAME;
+    $lfs_pass = Filesystem::readRandomCharacters(32);
+    $lfs_hash = PhabricatorHash::digest($lfs_pass);
+
+    $ttl = PhabricatorTime::getNow() + phutil_units('1 day in seconds');
+
+    $token = id(new PhabricatorAuthTemporaryToken())
+      ->setTokenResource($repository->getPHID())
+      ->setTokenType(self::TOKENTYPE)
+      ->setTokenCode($lfs_hash)
+      ->setUserPHID($viewer->getPHID())
+      ->setTemporaryTokenProperty('lfs.operation', $operation)
+      ->setTokenExpires($ttl)
+      ->save();
+
+    $authorization_header = base64_encode($lfs_user.':'.$lfs_pass);
+    return 'Basic '.$authorization_header;
+  }
+
 }

src/applications/repository/query/PhabricatorRepositoryGitLFSRefQuery.php

@@ -0,0 +1,64 @@
+<?php
+
+final class PhabricatorRepositoryGitLFSRefQuery
+  extends PhabricatorCursorPagedPolicyAwareQuery {
+
+  private $ids;
+  private $repositoryPHIDs;
+  private $objectHashes;
+
+  public function withIDs(array $ids) {
+    $this->ids = $ids;
+    return $this;
+  }
+
+  public function withRepositoryPHIDs(array $phids) {
+    $this->repositoryPHIDs = $phids;
+    return $this;
+  }
+
+  public function withObjectHashes(array $hashes) {
+    $this->objectHashes = $hashes;
+    return $this;
+  }
+
+  public function newResultObject() {
+    return new PhabricatorRepositoryGitLFSRef();
+  }
+
+  protected function loadPage() {
+    return $this->loadStandardPage($this->newResultObject());
+  }
+
+  protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) {
+    $where = parent::buildWhereClauseParts($conn);
+
+    if ($this->ids !== null) {
+      $where[] = qsprintf(
+        $conn,
+        'id IN (%Ld)',
+        $this->ids);
+    }
+
+    if ($this->repositoryPHIDs !== null) {
+      $where[] = qsprintf(
+        $conn,
+        'repositoryPHID IN (%Ls)',
+        $this->repositoryPHIDs);
+    }
+
+    if ($this->objectHashes !== null) {
+      $where[] = qsprintf(
+        $conn,
+        'objectHash IN (%Ls)',
+        $this->objectHashes);
+    }
+
+    return $where;
+  }
+
+  public function getQueryApplicationClass() {
+    return 'PhabricatorDiffusionApplication';
+  }
+
+}

src/applications/repository/storage/PhabricatorRepositoryGitLFSRef.php

@@ -0,0 +1,51 @@
+<?php
+
+final class PhabricatorRepositoryGitLFSRef
+  extends PhabricatorRepositoryDAO
+  implements PhabricatorPolicyInterface {
+
+  protected $repositoryPHID;
+  protected $objectHash;
+  protected $byteSize;
+  protected $authorPHID;
+  protected $filePHID;
+
+  protected function getConfiguration() {
+    return array(
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'objectHash' => 'bytes64',
+        'byteSize' => 'uint64',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_hash' => array(
+          'columns' => array('repositoryPHID', 'objectHash'),
+          'unique' => true,
+        ),
+      ),
+    ) + parent::getConfiguration();
+  }
+
+
+/* -(  PhabricatorPolicyInterface  )----------------------------------------- */
+
+
+  public function getCapabilities() {
+    return array(
+      PhabricatorPolicyCapability::CAN_VIEW,
+    );
+  }
+
+  public function getPolicy($capability) {
+    return PhabricatorPolicies::getMostOpenPolicy();
+  }
+
+  public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
+    return false;
+  }
+
+  public function describeAutomaticCapability($capability) {
+    return null;
+  }
+
+
+}