Use file.allocate to upload large files from arc diff

Summary: Fixes T8259. Depends on D13016. Use modern logic to support large file uploads.

Test Plan:
  - Diffed with some images, saw them show up in the diff.
  - Diffed with a 12MB binary, saw it upload in chunks.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T8259

Differential Revision: https://secure.phabricator.com/D13017

src/upload/ArcanistFileUploader.php

@@ -45,10 +45,27 @@ final class ArcanistFileUploader extends Phobject {
 
 
   /**
+   *
+   * @param ArcanistFileDataRef File data to upload.
+   * @param null|string Optional key to use to identify this file.
+   * @return this
    * @task add
    */
-  public function addFile(ArcanistFileDataRef $file) {
+  public function addFile(ArcanistFileDataRef $file, $key = null) {
-    $this->files[] = $file;
+
+    if ($key === null) {
+      $this->files[] = $file;
+    } else {
+      if (isset($this->files[$key])) {
+        throw new Exception(
+          pht(
+            'Two files were added with identical explicit keys ("%s"); each '.
+            'explicit key must be unique.',
+            $key));
+      }
+      $this->files[$key] = $file;
+    }
+
     return $this;
   }
 
@@ -57,6 +74,11 @@ final class ArcanistFileUploader extends Phobject {
 
 
   /**
+   * This method returns a map of all file data references. If references were
+   * added with an explicit key when @{method:addFile} was called, the key is
+   * retained in the result map.
+   *
+   * @return map<string, ArcanistFileDataRef> Files with results populated.
    * @task upload
    */
   public function uploadFiles() {

src/workflow/ArcanistDiffWorkflow.php

@@ -2544,72 +2544,51 @@ EOTEXT
       $change->setMetadata("{$type}:file:mime-type", $mime);
     }
 
-    echo pht('Uploading %d files...', count($need_upload))."\n";
+    $uploader = id(new ArcanistFileUploader())
+      ->setConduitClient($this->getConduit());
 
-    $hash_futures = array();
     foreach ($need_upload as $key => $spec) {
-      $hash_futures[$key] = $this->getConduit()->callMethod(
+      $ref = id(new ArcanistFileDataRef())
-        'file.uploadhash',
+        ->setName($spec['name'])
-        array(
+        ->setData($spec['data']);
-          'name' => $spec['name'],
+
-          'hash' => sha1($spec['data']),
+      $uploader->addFile($ref, $key);
-        ));
     }
 
-    $futures = id(new FutureIterator($hash_futures))
+    $files = $uploader->uploadFiles();
-      ->limit(8);
-    foreach ($futures as $key => $future) {
-      $type = $need_upload[$key]['type'];
-      $change = $need_upload[$key]['change'];
-      $name = $need_upload[$key]['name'];
 
-      $phid = null;
+    $errors = false;
-      try {
+    foreach ($files as $key => $file) {
-        $phid = $future->resolve();
+      if ($file->getErrors()) {
-      } catch (Exception $e) {
+        unset($files[$key]);
-        // Just try uploading normally if the hash upload failed.
+        $errors = true;
-        continue;
+        echo pht(
-      }
+          'Failed to upload binary "%s".',
-
+          $file->getName());
-      if ($phid) {
-        $change->setMetadata("{$type}:binary-phid", $phid);
-        unset($need_upload[$key]);
-        echo pht("Uploaded '%s' (%s).", $name, $type)."\n";
       }
     }
 
-    $upload_futures = array();
+    if ($errors) {
-    foreach ($need_upload as $key => $spec) {
+      $prompt = pht('Continue?');
-      $upload_futures[$key] = $this->getConduit()->callMethod(
+      $ok = phutil_console_confirm($prompt, $default_no = false);
-        'file.upload',
+      if (!$ok) {
-        array(
+        throw new ArcanistUsageException(
-          'name' => $spec['name'],
+          pht(
-          'data_base64' => base64_encode($spec['data']),
+            'Aborted due to file upload failure. You can use %s '.
-        ));
+            'to skip binary uploads.',
+            '--skip-binaries'));
+      }
     }
 
-    $futures = id(new FutureIterator($upload_futures))
+    foreach ($files as $key => $file) {
-      ->limit(4);
+      $spec = $need_upload[$key];
-    foreach ($futures as $key => $future) {
+      $phid = $file->getPHID();
-      $type = $need_upload[$key]['type'];
-      $change = $need_upload[$key]['change'];
-      $name = $need_upload[$key]['name'];
 
-      try {
+      $change = $spec['change'];
-        $phid = $future->resolve();
+      $type = $spec['type'];
-        $change->setMetadata("{$type}:binary-phid", $phid);
+      $change->setMetadata("{$type}:binary-phid", $phid);
-        echo pht("Uploaded '%s' (%s).", $name, $type)."\n";
+
-      } catch (Exception $e) {
+      echo pht('Uploaded binary data for "%s".', $file->getName())."\n";
-        echo pht("Failed to upload %s binary '%s'.", $type, $name)."\n\n";
-        echo $e->getMessage()."\n";
-        if (!phutil_console_confirm(pht('Continue?'), $default_no = false)) {
-          throw new ArcanistUsageException(
-            pht(
-              'Aborted due to file upload failure. You can use %s '.
-              'to skip binary uploads.',
-              '--skip-binaries'));
-        }
-      }
     }
 
     echo pht('Upload complete.')."\n";