Make "arc upload" chunk-aware

Summary: Ref T7149. This makes the client try to use the new `file.allocate` API before falling back to the old stuff.

Test Plan: Used `arc upload` to upload files. With chunking forced, uploaded chunked files.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: joshuaspence, epriestley

Maniphest Tasks: T7149

Differential Revision: https://secure.phabricator.com/D12061

src/workflow/ArcanistUploadWorkflow.php

@@ -55,24 +55,74 @@ EOTEXT
     $results = array();
 
     foreach ($this->paths as $path) {
-      $name = basename($path);
+      $path = Filesystem::resolvePath($path);
-      $this->writeStatusMessage(pht("Uploading '%s'...", $name)."\n");
 
+      $name = basename($path);
+      $this->writeStatus(pht("Uploading '%s'...", $name));
+
+      $hash = @sha1_file($path);
+      if (!$hash) {
+        throw new Exception(pht('Unable to read file "%s"!', $path));
+      }
+      $length = filesize($path);
+
+      $phid = null;
       try {
-        $data = Filesystem::readFile($path);
+        $result = $conduit->callMethodSynchronous(
-      } catch (FilesystemException $ex) {
+          'file.allocate',
-        $this->writeStatusMessage(
+          array(
-          pht('Unable to upload file: %s.', $ex->getMessage())."\n");
+            'name' => $name,
-        $results[$path] = null;
+            'contentLength' => $length,
-        continue;
+            'contentHash' => $hash,
+
+            // TODO: Remove this once this feature is good to go. For now,
+            // this is helpful for testing.
+            // 'forceChunking' => true,
+          ));
+
+        $phid = $result['filePHID'];
+        if (!$result['upload']) {
+          if (!$phid) {
+            $this->writeStatus(
+              pht(
+                'Unable to upload file "%s": the server refused to accept '.
+                'it. This usually means it is too large.',
+                $name));
+            continue;
+          }
+          // Otherwise, the server completed the upload by referencing known
+          // file data.
+        } else {
+          if ($phid) {
+            $this->uploadChunks($phid, $path);
+          } else {
+            // This is a small file that doesn't need to be uploaded in
+            // chunks, so continue normally.
+          }
+        }
+      } catch (Exception $ex) {
+        $this->writeStatus(
+          pht('Unable to use allocate method, trying older upload method.'));
+      }
+
+      if (!$phid) {
+        try {
+          $data = Filesystem::readFile($path);
+        } catch (FilesystemException $ex) {
+          $this->writeStatus(
+            pht('Unable to read file "%s".', $ex->getMessage()));
+          $results[$path] = null;
+          continue;
+        }
+
+        $phid = $conduit->callMethodSynchronous(
+          'file.upload',
+          array(
+            'data_base64' => base64_encode($data),
+            'name' => $name,
+          ));
       }
 
-      $phid = $conduit->callMethodSynchronous(
-        'file.upload',
-        array(
-          'data_base64' => base64_encode($data),
-          'name' => $name,
-        ));
       $info = $conduit->callMethodSynchronous(
         'file.info',
         array(
@@ -90,10 +140,85 @@ EOTEXT
     if ($this->json) {
       echo json_encode($results)."\n";
     } else {
-      $this->writeStatusMessage(pht('Done.')."\n");
+      $this->writeStatus(pht('Done.'));
     }
 
     return 0;
   }
 
+  private function writeStatus($line) {
+    $this->writeStatusMessage($line."\n");
+  }
+
+  private function uploadChunks($file_phid, $path) {
+    $conduit = $this->getConduit();
+
+    $f = @fopen($path, 'rb');
+    if (!$f) {
+      throw new Exception(pht('Unable to open file "%s"', $path));
+    }
+
+    $this->writeStatus(pht('Beginning chunked upload of large file...'));
+    $chunks = $conduit->callMethodSynchronous(
+      'file.querychunks',
+      array(
+        'filePHID' => $file_phid,
+      ));
+
+    $remaining = array();
+    foreach ($chunks as $chunk) {
+      if (!$chunk['complete']) {
+        $remaining[] = $chunk;
+      }
+    }
+
+    $done = (count($chunks) - count($remaining));
+
+    if ($done) {
+      $this->writeStatus(
+        pht(
+          'Resuming upload (%d of %d chunks remain).',
+          new PhutilNumber(count($remaining)),
+          new PhutilNumber(count($chunks))));
+    } else {
+      $this->writeStatus(
+        pht(
+          'Uploading chunks (%d chunks to upload).',
+          new PhutilNumber(count($remaining))));
+    }
+
+    $progress = new PhutilConsoleProgressBar();
+    $progress->setTotal(count($chunks));
+
+    for ($ii = 0; $ii < $done; $ii++) {
+      $progress->update(1);
+    }
+
+    // TODO: We could do these in parallel to improve upload performance.
+    foreach ($remaining as $chunk) {
+      $offset = $chunk['byteStart'];
+
+      $ok = fseek($f, $offset);
+      if ($ok !== 0) {
+        throw new Exception(pht('Failed to fseek()!'));
+      }
+
+      $data = fread($f, $chunk['byteEnd'] - $chunk['byteStart']);
+      if ($data === false) {
+        throw new Exception(pht('Failed to fread()!'));
+      }
+
+      $conduit->callMethodSynchronous(
+        'file.uploadchunk',
+        array(
+          'filePHID' => $file_phid,
+          'byteStart' => $offset,
+          'dataEncoding' => 'base64',
+          'data' => base64_encode($data),
+        ));
+
+      $progress->update(1);
+    }
+  }
+
 }