1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-12-19 20:10:55 +01:00

Support HTML5 / Javascript chunked file uploads

Summary:
Ref T7149. This adds chunking support to drag-and-drop uploads. It never activates right now unless you hack things up, since the chunk engine is still hard-coded as disabled.

The overall approach is the same as `arc upload` in D12061, with some slight changes to the API return values to avoid a few extra HTTP calls.

Test Plan:
  - Enabled chunk engine.
  - Uploaded some READMEs in a bunch of tiny 32 byte chunks.
  - Worked out of the box in Safari, Chrome, Firefox.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T7149

Differential Revision: https://secure.phabricator.com/D12066
This commit is contained in:
epriestley 2015-03-13 11:30:36 -07:00
parent aa4adf3ab8
commit 135280be9e
11 changed files with 492 additions and 113 deletions

View file

@ -8,10 +8,10 @@
return array(
'names' => array(
'core.pkg.css' => 'efdeeb14',
'core.pkg.js' => 'deae6907',
'core.pkg.js' => '31bc6546',
'darkconsole.pkg.js' => '8ab24e01',
'differential.pkg.css' => '1940be3f',
'differential.pkg.js' => '53c1ccc2',
'differential.pkg.js' => 'be1e5f9b',
'diffusion.pkg.css' => '591664fa',
'diffusion.pkg.js' => 'bfc0737b',
'maniphest.pkg.css' => '68d4dd3d',
@ -438,9 +438,9 @@ return array(
'rsrc/js/application/uiexample/gesture-example.js' => '558829c2',
'rsrc/js/application/uiexample/notification-example.js' => '8ce821c5',
'rsrc/js/core/Busy.js' => '6453c869',
'rsrc/js/core/DragAndDropFileUpload.js' => '8c49f386',
'rsrc/js/core/DragAndDropFileUpload.js' => 'fd6ace61',
'rsrc/js/core/DraggableList.js' => 'a16ec1c6',
'rsrc/js/core/FileUpload.js' => 'a4ae61bf',
'rsrc/js/core/FileUpload.js' => '477359c8',
'rsrc/js/core/Hovercard.js' => '7e8468ae',
'rsrc/js/core/KeyboardShortcut.js' => '1ae869f2',
'rsrc/js/core/KeyboardShortcutManager.js' => 'c1700f6f',
@ -458,13 +458,13 @@ return array(
'rsrc/js/core/behavior-crop.js' => 'fa0f4fc2',
'rsrc/js/core/behavior-dark-console.js' => '08883e8b',
'rsrc/js/core/behavior-device.js' => '03d6ed07',
'rsrc/js/core/behavior-drag-and-drop-textarea.js' => '92eb531d',
'rsrc/js/core/behavior-drag-and-drop-textarea.js' => '6d49590e',
'rsrc/js/core/behavior-error-log.js' => '6882e80a',
'rsrc/js/core/behavior-fancy-datepicker.js' => 'c51ae228',
'rsrc/js/core/behavior-file-tree.js' => '88236f00',
'rsrc/js/core/behavior-form.js' => '5c54cbf3',
'rsrc/js/core/behavior-gesture.js' => '3ab51e2c',
'rsrc/js/core/behavior-global-drag-and-drop.js' => '8c584f17',
'rsrc/js/core/behavior-global-drag-and-drop.js' => 'bbdf75ca',
'rsrc/js/core/behavior-high-security-warning.js' => '8fc1c918',
'rsrc/js/core/behavior-history-install.js' => '7ee2b591',
'rsrc/js/core/behavior-hovercard.js' => 'f36e01af',
@ -549,7 +549,7 @@ return array(
'javelin-behavior-aphlict-status' => 'ea681761',
'javelin-behavior-aphront-basic-tokenizer' => 'b3a4b884',
'javelin-behavior-aphront-crop' => 'fa0f4fc2',
'javelin-behavior-aphront-drag-and-drop-textarea' => '92eb531d',
'javelin-behavior-aphront-drag-and-drop-textarea' => '6d49590e',
'javelin-behavior-aphront-form-disable-on-submit' => '5c54cbf3',
'javelin-behavior-aphront-more' => 'a80d0378',
'javelin-behavior-audio-source' => '59b251eb',
@ -588,7 +588,7 @@ return array(
'javelin-behavior-durable-column' => 'a3ba7034',
'javelin-behavior-error-log' => '6882e80a',
'javelin-behavior-fancy-datepicker' => 'c51ae228',
'javelin-behavior-global-drag-and-drop' => '8c584f17',
'javelin-behavior-global-drag-and-drop' => 'bbdf75ca',
'javelin-behavior-herald-rule-editor' => '7ebaeed3',
'javelin-behavior-high-security-warning' => '8fc1c918',
'javelin-behavior-history-install' => '7ee2b591',
@ -719,11 +719,11 @@ return array(
'phabricator-core-css' => '86bfbe8c',
'phabricator-countdown-css' => '86b7b0a0',
'phabricator-dashboard-css' => '17937d22',
'phabricator-drag-and-drop-file-upload' => '8c49f386',
'phabricator-drag-and-drop-file-upload' => 'fd6ace61',
'phabricator-draggable-list' => 'a16ec1c6',
'phabricator-fatal-config-template-css' => '8e6c6fcd',
'phabricator-feed-css' => 'b513b5f4',
'phabricator-file-upload' => 'a4ae61bf',
'phabricator-file-upload' => '477359c8',
'phabricator-filetree-view-css' => 'fccf9f82',
'phabricator-flag-css' => '5337623f',
'phabricator-hovercard' => '7e8468ae',
@ -1122,6 +1122,11 @@ return array(
'javelin-dom',
'javelin-workflow',
),
'477359c8' => array(
'javelin-install',
'javelin-dom',
'phabricator-notification',
),
47830651 => array(
'javelin-behavior',
'javelin-dom',
@ -1273,6 +1278,12 @@ return array(
'javelin-typeahead',
'javelin-uri',
),
'6d49590e' => array(
'javelin-behavior',
'javelin-dom',
'phabricator-drag-and-drop-file-upload',
'phabricator-textareautils',
),
'6e2de6f2' => array(
'multirow-row-manager',
'javelin-install',
@ -1508,21 +1519,6 @@ return array(
'javelin-request',
'javelin-typeahead-source',
),
'8c49f386' => array(
'javelin-install',
'javelin-util',
'javelin-request',
'javelin-dom',
'javelin-uri',
'phabricator-file-upload',
),
'8c584f17' => array(
'javelin-behavior',
'javelin-dom',
'javelin-uri',
'javelin-mask',
'phabricator-drag-and-drop-file-upload',
),
'8ce821c5' => array(
'phabricator-notification',
'javelin-stratcom',
@ -1546,12 +1542,6 @@ return array(
'javelin-uri',
'phabricator-notification',
),
'92eb531d' => array(
'javelin-behavior',
'javelin-dom',
'phabricator-drag-and-drop-file-upload',
'phabricator-textareautils',
),
'9414ff18' => array(
'javelin-behavior',
'javelin-resource',
@ -1638,11 +1628,6 @@ return array(
'javelin-vector',
'differential-inline-comment-editor',
),
'a4ae61bf' => array(
'javelin-install',
'javelin-dom',
'phabricator-notification',
),
'a80d0378' => array(
'javelin-behavior',
'javelin-stratcom',
@ -1718,6 +1703,13 @@ return array(
'javelin-stratcom',
'javelin-dom',
),
'bbdf75ca' => array(
'javelin-behavior',
'javelin-dom',
'javelin-uri',
'javelin-mask',
'phabricator-drag-and-drop-file-upload',
),
'bd4c8dca' => array(
'javelin-install',
'javelin-util',
@ -2007,6 +1999,14 @@ return array(
'javelin-dom',
'phortune-credit-card-form',
),
'fd6ace61' => array(
'javelin-install',
'javelin-util',
'javelin-request',
'javelin-dom',
'javelin-uri',
'phabricator-file-upload',
),
'fe287620' => array(
'javelin-install',
'javelin-dom',

View file

@ -121,10 +121,20 @@ final class FileAllocateConduitAPIMethod
}
// None of the storage engines can accept this file.
if (PhabricatorFileStorageEngine::loadWritableEngines()) {
$error = pht(
'Unable to upload file: this file is too large for any '.
'configured storage engine.');
} else {
$error = pht(
'Unable to upload file: the server is not configured with any '.
'writable storage engines.');
}
return array(
'upload' => false,
'filePHID' => null,
'error' => $error,
);
}

View file

@ -13,18 +13,82 @@ final class PhabricatorFileDropUploadController
// NOTE: Throws if valid CSRF token is not present in the request.
$request->validateCSRF();
$data = PhabricatorStartup::getRawInput();
$name = $request->getStr('name');
$file_phid = $request->getStr('phid');
// If there's no explicit view policy, make it very restrictive by default.
// This is the correct policy for files dropped onto objects during
// creation, comment and edit flows.
$view_policy = $request->getStr('viewPolicy');
if (!$view_policy) {
$view_policy = $viewer->getPHID();
}
$is_chunks = $request->getBool('querychunks');
if ($is_chunks) {
$params = array(
'filePHID' => $file_phid,
);
$result = id(new ConduitCall('file.querychunks', $params))
->setUser($viewer)
->execute();
return id(new AphrontAjaxResponse())->setContent($result);
}
$is_allocate = $request->getBool('allocate');
if ($is_allocate) {
$params = array(
'name' => $name,
'contentLength' => $request->getInt('length'),
'viewPolicy' => $view_policy,
// TODO: Remove.
// 'forceChunking' => true,
);
$result = id(new ConduitCall('file.allocate', $params))
->setUser($viewer)
->execute();
$file_phid = $result['filePHID'];
if ($file_phid) {
$file = $this->loadFile($file_phid);
$result += $this->getFileDictionary($file);
}
return id(new AphrontAjaxResponse())->setContent($result);
}
// Read the raw request data. We're either doing a chunk upload or a
// vanilla upload, so we need it.
$data = PhabricatorStartup::getRawInput();
$is_chunk_upload = $request->getBool('uploadchunk');
if ($is_chunk_upload) {
$params = array(
'filePHID' => $file_phid,
'byteStart' => $request->getInt('byteStart'),
'data' => $data,
);
$result = id(new ConduitCall('file.uploadchunk', $params))
->setUser($viewer)
->execute();
$file = $this->loadFile($file_phid);
if ($file->getIsPartial()) {
$result = array();
} else {
$result = array(
'complete' => true,
) + $this->getFileDictionary($file);
}
return id(new AphrontAjaxResponse())->setContent($result);
}
$file = PhabricatorFile::newFromXHRUpload(
$data,
array(
@ -34,12 +98,30 @@ final class PhabricatorFileDropUploadController
'isExplicitUpload' => true,
));
return id(new AphrontAjaxResponse())->setContent(
array(
'id' => $file->getID(),
'phid' => $file->getPHID(),
'uri' => $file->getBestURI(),
));
$result = $this->getFileDictionary($file);
return id(new AphrontAjaxResponse())->setContent($result);
}
private function getFileDictionary(PhabricatorFile $file) {
return array(
'id' => $file->getID(),
'phid' => $file->getPHID(),
'uri' => $file->getBestURI(),
);
}
private function loadFile($file_phid) {
$viewer = $this->getViewer();
$file = id(new PhabricatorFileQuery())
->setViewer($viewer)
->withPHIDs(array($file_phid))
->executeOne();
if (!$file) {
throw new Exception(pht('Failed to load file.'));
}
return $file;
}
}

View file

@ -162,7 +162,7 @@ final class PhabricatorChunkedFileStorageEngine
return false;
}
private function getChunkSize() {
public function getChunkSize() {
// TODO: This is an artificially small size to make it easier to
// test chunking.
return 32;

View file

@ -255,4 +255,40 @@ abstract class PhabricatorFileStorageEngine {
return $writable;
}
/**
* Return the largest file size which can be uploaded without chunking.
*
* Files smaller than this will always upload in one request, so clients
* can safely skip the allocation step.
*
* @return int|null Byte size, or `null` if there is no chunk support.
*/
public static function getChunkThreshold() {
$engines = self::loadWritableEngines();
$min = null;
foreach ($engines as $engine) {
if (!$engine->isChunkEngine()) {
continue;
}
if (!$min) {
$min = $engine;
continue;
}
if ($min->getChunkSize() > $engine->getChunkSize()) {
$min = $engine->getChunkSize();
}
}
if (!$min) {
return null;
}
return $engine->getChunkSize();
}
}

View file

@ -24,11 +24,12 @@ final class PhabricatorGlobalUploadTargetView extends AphrontView {
require_celerity_resource('global-drag-and-drop-css');
Javelin::initBehavior('global-drag-and-drop', array(
'ifSupported' => $this->showIfSupportedID,
'instructions' => $instructions_id,
'uploadURI' => '/file/dropupload/',
'browseURI' => '/file/query/authored/',
'viewPolicy' => PhabricatorPolicies::getMostOpenPolicy(),
'ifSupported' => $this->showIfSupportedID,
'instructions' => $instructions_id,
'uploadURI' => '/file/dropupload/',
'browseURI' => '/file/query/authored/',
'viewPolicy' => PhabricatorPolicies::getMostOpenPolicy(),
'chunkThreshold' => PhabricatorFileStorageEngine::getChunkThreshold(),
));
return phutil_tag(

View file

@ -35,9 +35,10 @@ final class PhabricatorRemarkupControl extends AphrontFormTextAreaControl {
Javelin::initBehavior(
'aphront-drag-and-drop-textarea',
array(
'target' => $id,
'activatedClass' => 'aphront-textarea-drag-and-drop',
'uri' => '/file/dropupload/',
'target' => $id,
'activatedClass' => 'aphront-textarea-drag-and-drop',
'uri' => '/file/dropupload/',
'chunkThreshold' => PhabricatorFileStorageEngine::getChunkThreshold(),
));
Javelin::initBehavior(

View file

@ -169,64 +169,189 @@ JX.install('PhabricatorDragAndDropFileUpload', {
}));
}
},
_sendRequest : function(spec) {
var file = new JX.PhabricatorFileUpload()
.setRawFileObject(spec)
.setName(spec.name)
.setTotalBytes(spec.size)
.setTotalBytes(spec.size);
var threshold = this.getChunkThreshold();
if (threshold && (file.getTotalBytes() > threshold)) {
// This is a large file, so we'll go through allocation so we can
// pick up support for resume and chunking.
this._allocateFile(file);
} else {
// If this file is smaller than the chunk threshold, skip the round
// trip for allocation and just upload it directly.
this._sendDataRequest(file);
}
},
_allocateFile: function(file) {
file
.setStatus('allocate')
.update();
var alloc_uri = this._getUploadURI(file)
.setQueryParam('allocate', 1);
new JX.Workflow(alloc_uri)
.setHandler(JX.bind(this, this._didAllocateFile, file))
.start();
},
_getUploadURI: function(file) {
var uri = JX.$U(this.getURI())
.setQueryParam('name', file.getName())
.setQueryParam('length', file.getTotalBytes());
if (this.getViewPolicy()) {
uri.setQueryParam('viewPolicy', this.getViewPolicy());
}
if (file.getAllocatedPHID()) {
uri.setQueryParam('phid', file.getAllocatedPHID());
}
return uri;
},
_didAllocateFile: function(file, r) {
var phid = r.phid;
var upload = r.upload;
if (!upload) {
if (phid) {
this._completeUpload(file, r);
} else {
this._failUpload(file, r);
}
return;
} else {
if (phid) {
// Start or resume a chunked upload.
file.setAllocatedPHID(phid);
this._loadChunks(file);
} else {
// Proceed with non-chunked upload.
this._sendDataRequest(file);
}
}
},
_loadChunks: function(file) {
file
.setStatus('chunks')
.update();
var chunks_uri = this._getUploadURI(file)
.setQueryParam('querychunks', 1);
new JX.Workflow(chunks_uri)
.setHandler(JX.bind(this, this._didLoadChunks, file))
.start();
},
_didLoadChunks: function(file, r) {
file.setChunks(r);
this._uploadNextChunk(file);
},
_uploadNextChunk: function(file) {
var chunks = file.getChunks();
var chunk;
for (var ii = 0; ii < chunks.length; ii++) {
chunk = chunks[ii];
if (!chunk.complete) {
this._readChunk(
file,
chunk,
JX.bind(this, this._didReadChunk, file, chunk));
break;
}
}
},
_readChunk: function(file, chunk, callback) {
var reader = new FileReader();
var blob = file.getRawFileObject().slice(chunk.byteStart, chunk.byteEnd);
reader.onload = function() {
callback(reader.result);
};
reader.onerror = function() {
this._failUpload(file, {error: reader.error.message});
};
reader.readAsBinaryString(blob);
},
_didReadChunk: function(file, chunk, data) {
file
.setStatus('upload')
.update();
var chunkup_uri = this._getUploadURI(file)
.setQueryParam('uploadchunk', 1)
.setQueryParam('__upload__', 1)
.setQueryParam('byteStart', chunk.byteStart)
.toString();
var callback = JX.bind(this, this._didUploadChunk, file, chunk);
var req = new JX.Request(chunkup_uri, callback);
var seen_bytes = 0;
var onprogress = JX.bind(this, function(progress) {
file
.addUploadedBytes(progress.loaded - seen_bytes)
.update();
seen_bytes = progress.loaded;
this.invoke('progress', file);
});
req.listen('error', JX.bind(this, this._onUploadError, req, file));
req.listen('uploadprogress', onprogress);
req
.setRawData(data)
.send();
},
_didUploadChunk: function(file, chunk, r) {
file.didCompleteChunk(chunk);
if (r.complete) {
this._completeUpload(file, r);
} else {
this._uploadNextChunk(file);
}
},
_sendDataRequest: function(file) {
file
.setStatus('uploading')
.update();
this.invoke('willUpload', file);
var up_uri = JX.$U(this.getURI())
.setQueryParam('name', file.getName())
.setQueryParam('__upload__', 1);
if (this.getViewPolicy()) {
up_uri.setQueryParam('viewPolicy', this.getViewPolicy());
}
up_uri = up_uri.toString();
var up_uri = this._getUploadURI(file)
.setQueryParam('__upload__', 1)
.toString();
var onupload = JX.bind(this, function(r) {
if (r.error) {
file
.setStatus('error')
.setError(r.error)
.update();
this.invoke('didError', file);
this._failUpload(file, r);
} else {
file
.setID(r.id)
.setPHID(r.phid)
.setURI(r.uri)
.setMarkup(r.html)
.setStatus('done')
.update();
this.invoke('didUpload', file);
this._completeUpload(file, r);
}
});
var req = new JX.Request(up_uri, onupload);
var onerror = JX.bind(this, function(error) {
file.setStatus('error');
if (error) {
file.setError(error.code + ': ' + error.info);
} else {
var xhr = req.getTransport();
if (xhr.responseText) {
file.setError('Server responded: ' + xhr.responseText);
}
}
file.update();
this.invoke('didError', file);
});
var onprogress = JX.bind(this, function(progress) {
file
.setTotalBytes(progress.total)
@ -236,17 +361,56 @@ JX.install('PhabricatorDragAndDropFileUpload', {
this.invoke('progress', file);
});
req.listen('error', onerror);
req.listen('error', JX.bind(this, this._onUploadError, req, file));
req.listen('uploadprogress', onprogress);
req
.setRawData(spec)
.setRawData(file.getRawFileObject())
.send();
},
_completeUpload: function(file, r) {
file
.setID(r.id)
.setPHID(r.phid)
.setURI(r.uri)
.setMarkup(r.html)
.setStatus('done')
.update();
this.invoke('didUpload', file);
},
_failUpload: function(file, r) {
file
.setStatus('error')
.setError(r.error)
.update();
this.invoke('didError', file);
},
_onUploadError: function(file, req, error) {
file.setStatus('error');
if (error) {
file.setError(error.code + ': ' + error.info);
} else {
var xhr = req.getTransport();
if (xhr.responseText) {
file.setError('Server responded: ' + xhr.responseText);
}
}
file.update();
this.invoke('didError', file);
}
},
properties: {
URI : null,
activatedClass : null,
viewPolicy : null
URI: null,
activatedClass: null,
viewPolicy: null,
chunkThreshold: null
}
});

View file

@ -13,19 +13,77 @@ JX.install('PhabricatorFileUpload', {
},
properties : {
name : null,
totalBytes : null,
uploadedBytes : null,
ID : null,
PHID : null,
URI : null,
status : null,
markup : null,
error : null
name: null,
totalBytes: null,
uploadedBytes: null,
rawFileObject: null,
allocatedPHID: null,
ID: null,
PHID: null,
URI: null,
status: null,
markup: null,
error: null
},
members : {
_notification : null,
_chunks: null,
_isResume: false,
addUploadedBytes: function(bytes) {
var uploaded = this.getUploadedBytes();
this.setUploadedBytes(uploaded + bytes);
return this;
},
setChunks: function(chunks) {
var chunk;
for (var ii = 0; ii < chunks.length; ii++) {
chunk = chunks[ii];
if (chunk.complete) {
this.addUploadedBytes(chunk.byteEnd - chunk.byteStart);
this._isResume = true;
}
}
this._chunks = chunks;
return this;
},
getChunks: function() {
return this._chunks;
},
getRemainingChunks: function() {
var chunks = this.getChunks();
var result = [];
for (var ii = 0; ii < chunks.length; ii++) {
if (!chunks[ii].complete) {
result.push(chunks[ii]);
}
}
return result;
},
didCompleteChunk: function(chunk) {
var chunks = this.getRemainingChunks();
for (var ii = 0; ii < chunks.length; ii++) {
if (chunks[ii].byteStart == chunk.byteStart) {
if (chunks[ii].byteEnd == chunk.byteEnd) {
if (!chunks[ii].complete) {
chunks[ii].complete = true;
}
break;
}
}
}
return this;
},
update : function() {
if (!this._notification) {
@ -37,6 +95,9 @@ JX.install('PhabricatorFileUpload', {
.show();
var content;
// TODO: This stuff needs some work for translations.
switch (this.getStatus()) {
case 'done':
var link = JX.$N('a', {href: this.getURI()}, 'F' + this.getID());
@ -68,15 +129,37 @@ JX.install('PhabricatorFileUpload', {
.alterClassName('jx-notification-error', true);
this._notification = null;
break;
case 'allocate':
content = 'Allocating "' + this.getName() + '"...';
this._notification
.setContent(content);
break;
case 'chunks':
content = 'Loading chunks for "' + this.getName() + '"...';
this._notification
.setContent(content);
break;
default:
var info = '';
if (this.getTotalBytes()) {
var p = this._renderPercentComplete();
var f = this._renderFileSize();
info = ' (' + p + ' of ' + f + ')';
info = p + ' of ' + f;
}
info = 'Uploading "' + this.getName() + '"' + info + '...';
var head;
if (this._isResume) {
head = 'Resuming:';
} else if (this._chunks) {
head = 'Uploading chunks:';
} else {
head = 'Uploading:';
}
info = [
JX.$N('strong', {}, this.getName()),
JX.$N('br'),
head + ' ' + info];
this._notification
.setContent(info);

View file

@ -27,7 +27,8 @@ JX.behavior('aphront-drag-and-drop-textarea', function(config) {
if (JX.PhabricatorDragAndDropFileUpload.isSupported()) {
var drop = new JX.PhabricatorDragAndDropFileUpload(target)
.setURI(config.uri);
.setURI(config.uri)
.setChunkThreshold(config.chunkThreshold);
drop.listen('didBeginDrag', function() {
JX.DOM.alterClass(target, config.activatedClass, true);
});

View file

@ -23,7 +23,8 @@ JX.behavior('global-drag-and-drop', function(config) {
var page = JX.$('phabricator-standard-page');
var drop = new JX.PhabricatorDragAndDropFileUpload(page)
.setURI(config.uploadURI)
.setViewPolicy(config.viewPolicy);
.setViewPolicy(config.viewPolicy)
.setChunkThreshold(config.chunkThreshold);
drop.listen('didBeginDrag', function() {
JX.Mask.show('global-upload-mask');