Use file.allocate to upload large files from arc diff

Summary: Fixes T8259. Depends on D13016. Use modern logic to support large file uploads.

Test Plan:
  - Diffed with some images, saw them show up in the diff.
  - Diffed with a 12MB binary, saw it upload in chunks.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T8259

Differential Revision: https://secure.phabricator.com/D13017

src/upload/ArcanistFileUploader.php

@@ -45,10 +45,27 @@ final class ArcanistFileUploader extends Phobject {
 
 
   /**
+   *
+   * @param ArcanistFileDataRef File data to upload.
+   * @param null|string Optional key to use to identify this file.
+   * @return this
    * @task add
    */
-  public function addFile(ArcanistFileDataRef $file) {
+  public function addFile(ArcanistFileDataRef $file, $key = null) {
+
+    if ($key === null) {
       $this->files[] = $file;
+    } else {
+      if (isset($this->files[$key])) {
+        throw new Exception(
+          pht(
+            'Two files were added with identical explicit keys ("%s"); each '.
+            'explicit key must be unique.',
+            $key));
+      }
+      $this->files[$key] = $file;
+    }
+
     return $this;
   }
 
@@ -57,6 +74,11 @@ final class ArcanistFileUploader extends Phobject {
 
 
   /**
+   * This method returns a map of all file data references. If references were
+   * added with an explicit key when @{method:addFile} was called, the key is
+   * retained in the result map.
+   *
+   * @return map<string, ArcanistFileDataRef> Files with results populated.
    * @task upload
    */
   public function uploadFiles() {

src/workflow/ArcanistDiffWorkflow.php

@@ -2544,65 +2544,34 @@ EOTEXT
       $change->setMetadata("{$type}:file:mime-type", $mime);
     }
 
-    echo pht('Uploading %d files...', count($need_upload))."\n";
+    $uploader = id(new ArcanistFileUploader())
+      ->setConduitClient($this->getConduit());
 
-    $hash_futures = array();
     foreach ($need_upload as $key => $spec) {
-      $hash_futures[$key] = $this->getConduit()->callMethod(
-        'file.uploadhash',
-        array(
-          'name' => $spec['name'],
-          'hash' => sha1($spec['data']),
-        ));
+      $ref = id(new ArcanistFileDataRef())
+        ->setName($spec['name'])
+        ->setData($spec['data']);
+
+      $uploader->addFile($ref, $key);
     }
 
-    $futures = id(new FutureIterator($hash_futures))
-      ->limit(8);
-    foreach ($futures as $key => $future) {
-      $type = $need_upload[$key]['type'];
-      $change = $need_upload[$key]['change'];
-      $name = $need_upload[$key]['name'];
+    $files = $uploader->uploadFiles();
 
-      $phid = null;
-      try {
-        $phid = $future->resolve();
-      } catch (Exception $e) {
-        // Just try uploading normally if the hash upload failed.
-        continue;
-      }
-
-      if ($phid) {
-        $change->setMetadata("{$type}:binary-phid", $phid);
-        unset($need_upload[$key]);
-        echo pht("Uploaded '%s' (%s).", $name, $type)."\n";
+    $errors = false;
+    foreach ($files as $key => $file) {
+      if ($file->getErrors()) {
+        unset($files[$key]);
+        $errors = true;
+        echo pht(
+          'Failed to upload binary "%s".',
+          $file->getName());
       }
     }
 
-    $upload_futures = array();
-    foreach ($need_upload as $key => $spec) {
-      $upload_futures[$key] = $this->getConduit()->callMethod(
-        'file.upload',
-        array(
-          'name' => $spec['name'],
-          'data_base64' => base64_encode($spec['data']),
-        ));
-    }
-
-    $futures = id(new FutureIterator($upload_futures))
-      ->limit(4);
-    foreach ($futures as $key => $future) {
-      $type = $need_upload[$key]['type'];
-      $change = $need_upload[$key]['change'];
-      $name = $need_upload[$key]['name'];
-
-      try {
-        $phid = $future->resolve();
-        $change->setMetadata("{$type}:binary-phid", $phid);
-        echo pht("Uploaded '%s' (%s).", $name, $type)."\n";
-      } catch (Exception $e) {
-        echo pht("Failed to upload %s binary '%s'.", $type, $name)."\n\n";
-        echo $e->getMessage()."\n";
-        if (!phutil_console_confirm(pht('Continue?'), $default_no = false)) {
+    if ($errors) {
+      $prompt = pht('Continue?');
+      $ok = phutil_console_confirm($prompt, $default_no = false);
+      if (!$ok) {
         throw new ArcanistUsageException(
           pht(
             'Aborted due to file upload failure. You can use %s '.
@@ -2610,6 +2579,16 @@ EOTEXT
             '--skip-binaries'));
       }
     }
+
+    foreach ($files as $key => $file) {
+      $spec = $need_upload[$key];
+      $phid = $file->getPHID();
+
+      $change = $spec['change'];
+      $type = $spec['type'];
+      $change->setMetadata("{$type}:binary-phid", $phid);
+
+      echo pht('Uploaded binary data for "%s".', $file->getName())."\n";
     }
 
     echo pht('Upload complete.')."\n";