revi-archive/phorge-arcanist
1
0
Fork 0
mirror of https://we.phorge.it/source/arcanist.git synced 2024-11-25 16:22:42 +01:00
Code Issues Releases Wiki Activity

Make "arc upload" chunk-aware

Browse source 
Summary: Ref T7149. This makes the client try to use the new `file.allocate` API before falling back to the old stuff.

Test Plan: Used `arc upload` to upload files. With chunking forced, uploaded chunked files.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: joshuaspence, epriestley

Maniphest Tasks: T7149

Differential Revision: https://secure.phabricator.com/D12061
This commit is contained in:
epriestley 2015-03-13 11:30:50 -07:00
parent f51f55a63c
commit a01d3c3b1a
1 changed files with 140 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

155
src/workflow/ArcanistUploadWorkflow.php
View file

@ -55,24 +55,74 @@ EOTEXT
$results = array();
foreach ($this->paths as $path) {
$name = basename($path);
$this->writeStatusMessage(pht("Uploading '%s'...", $name)."\n");
$path = Filesystem::resolvePath($path);
$name = basename($path);
$this->writeStatus(pht("Uploading '%s'...", $name));
$hash = @sha1_file($path);
if (!$hash) {
throw new Exception(pht('Unable to read file "%s"!', $path));
}
$length = filesize($path);
$phid = null;
try {
$data = Filesystem::readFile($path);
} catch (FilesystemException $ex) {
$this->writeStatusMessage(
pht('Unable to upload file: %s.', $ex->getMessage())."\n");
$results[$path] = null;
continue;
$result = $conduit->callMethodSynchronous(
'file.allocate',
array(
'name' => $name,
'contentLength' => $length,
'contentHash' => $hash,
// TODO: Remove this once this feature is good to go. For now,
// this is helpful for testing.
// 'forceChunking' => true,
));
$phid = $result['filePHID'];
if (!$result['upload']) {
if (!$phid) {
$this->writeStatus(
pht(
'Unable to upload file "%s": the server refused to accept '.
'it. This usually means it is too large.',
$name));
continue;
}
// Otherwise, the server completed the upload by referencing known
// file data.
} else {
if ($phid) {
$this->uploadChunks($phid, $path);
} else {
// This is a small file that doesn't need to be uploaded in
// chunks, so continue normally.
}
}
} catch (Exception $ex) {
$this->writeStatus(
pht('Unable to use allocate method, trying older upload method.'));
}
if (!$phid) {
try {
$data = Filesystem::readFile($path);
} catch (FilesystemException $ex) {
$this->writeStatus(
pht('Unable to read file "%s".', $ex->getMessage()));
$results[$path] = null;
continue;
}
$phid = $conduit->callMethodSynchronous(
'file.upload',
array(
'data_base64' => base64_encode($data),
'name' => $name,
));
}
$phid = $conduit->callMethodSynchronous(
'file.upload',
array(
'data_base64' => base64_encode($data),
'name' => $name,
));
$info = $conduit->callMethodSynchronous(
'file.info',
array(
@ -90,10 +140,85 @@ EOTEXT
if ($this->json) {
echo json_encode($results)."\n";
} else {
$this->writeStatusMessage(pht('Done.')."\n");
$this->writeStatus(pht('Done.'));
}
return 0;
}
private function writeStatus($line) {
$this->writeStatusMessage($line."\n");
}
private function uploadChunks($file_phid, $path) {
$conduit = $this->getConduit();
$f = @fopen($path, 'rb');
if (!$f) {
throw new Exception(pht('Unable to open file "%s"', $path));
}
$this->writeStatus(pht('Beginning chunked upload of large file...'));
$chunks = $conduit->callMethodSynchronous(
'file.querychunks',
array(
'filePHID' => $file_phid,
));
$remaining = array();
foreach ($chunks as $chunk) {
if (!$chunk['complete']) {
$remaining[] = $chunk;
}
}
$done = (count($chunks) - count($remaining));
if ($done) {
$this->writeStatus(
pht(
'Resuming upload (%d of %d chunks remain).',
new PhutilNumber(count($remaining)),
new PhutilNumber(count($chunks))));
} else {
$this->writeStatus(
pht(
'Uploading chunks (%d chunks to upload).',
new PhutilNumber(count($remaining))));
}
$progress = new PhutilConsoleProgressBar();
$progress->setTotal(count($chunks));
for ($ii = 0; $ii < $done; $ii++) {
$progress->update(1);
}
// TODO: We could do these in parallel to improve upload performance.
foreach ($remaining as $chunk) {
$offset = $chunk['byteStart'];
$ok = fseek($f, $offset);
if ($ok !== 0) {
throw new Exception(pht('Failed to fseek()!'));
}
$data = fread($f, $chunk['byteEnd'] - $chunk['byteStart']);
if ($data === false) {
throw new Exception(pht('Failed to fread()!'));
}
$conduit->callMethodSynchronous(
'file.uploadchunk',
array(
'filePHID' => $file_phid,
'byteStart' => $offset,
'dataEncoding' => 'base64',
'data' => base64_encode($data),
));
$progress->update(1);
}
}
}