1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-27 09:12:41 +01:00

Support resuming JS uploads of chunked files

Summary: Ref T7149. We can't compute hashes of large files efficiently, but we can resume uploads by the same author, with the same name and file size, which are only partially completed. This seems like a reasonable heuristic that is unlikely to ever misfire, even if it's a little magical.

Test Plan:
  - Forced chunking on.
  - Started uploading a chunked file.
  - Closed the browser window.
  - Dropped it into a new window.
  - Upload resumed //(!!!)//
  - Did this again.
  - Downloaded the final file, which successfully reconstructed the original file.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: joshuaspence, chad, epriestley

Maniphest Tasks: T7149

Differential Revision: https://secure.phabricator.com/D12070
This commit is contained in:
epriestley 2015-03-14 08:28:46 -07:00
parent aa909ba072
commit 32d8d67535
5 changed files with 101 additions and 30 deletions

View file

@ -44,7 +44,7 @@ return array(
'rsrc/css/application/config/config-welcome.css' => '6abd79be', 'rsrc/css/application/config/config-welcome.css' => '6abd79be',
'rsrc/css/application/config/setup-issue.css' => '22270af2', 'rsrc/css/application/config/setup-issue.css' => '22270af2',
'rsrc/css/application/config/unhandled-exception.css' => '37d4f9a2', 'rsrc/css/application/config/unhandled-exception.css' => '37d4f9a2',
'rsrc/css/application/conpherence/durable-column.css' => '1f5c64e8', 'rsrc/css/application/conpherence/durable-column.css' => '8c951609',
'rsrc/css/application/conpherence/menu.css' => 'c6ac5299', 'rsrc/css/application/conpherence/menu.css' => 'c6ac5299',
'rsrc/css/application/conpherence/message-pane.css' => '5930260a', 'rsrc/css/application/conpherence/message-pane.css' => '5930260a',
'rsrc/css/application/conpherence/notification.css' => '04a6e10a', 'rsrc/css/application/conpherence/notification.css' => '04a6e10a',
@ -513,7 +513,7 @@ return array(
'changeset-view-manager' => '88be0133', 'changeset-view-manager' => '88be0133',
'config-options-css' => '7fedf08b', 'config-options-css' => '7fedf08b',
'config-welcome-css' => '6abd79be', 'config-welcome-css' => '6abd79be',
'conpherence-durable-column-view' => '1f5c64e8', 'conpherence-durable-column-view' => '8c951609',
'conpherence-menu-css' => 'c6ac5299', 'conpherence-menu-css' => 'c6ac5299',
'conpherence-message-pane-css' => '5930260a', 'conpherence-message-pane-css' => '5930260a',
'conpherence-notification-css' => '04a6e10a', 'conpherence-notification-css' => '04a6e10a',

View file

@ -37,7 +37,7 @@ final class FileAllocateConduitAPIMethod
$hash = $request->getValue('contentHash'); $hash = $request->getValue('contentHash');
$name = $request->getValue('name'); $name = $request->getValue('name');
$view_policy = $request->getValue('viewPolicy'); $view_policy = $request->getValue('viewPolicy');
$content_length = $request->getValue('contentLength'); $length = $request->getValue('contentLength');
$force_chunking = $request->getValue('forceChunking'); $force_chunking = $request->getValue('forceChunking');
@ -48,18 +48,14 @@ final class FileAllocateConduitAPIMethod
'isExplicitUpload' => true, 'isExplicitUpload' => true,
); );
$file = null;
if ($hash) { if ($hash) {
$file = PhabricatorFile::newFileFromContentHash( $file = PhabricatorFile::newFileFromContentHash(
$hash, $hash,
$properties); $properties);
if ($file) {
return array(
'upload' => false,
'filePHID' => $file->getPHID(),
);
} }
if ($hash && !$file) {
$chunked_hash = PhabricatorChunkedFileStorageEngine::getChunkedHash( $chunked_hash = PhabricatorChunkedFileStorageEngine::getChunkedHash(
$viewer, $viewer,
$hash); $hash);
@ -67,6 +63,25 @@ final class FileAllocateConduitAPIMethod
->setViewer($viewer) ->setViewer($viewer)
->withContentHashes(array($chunked_hash)) ->withContentHashes(array($chunked_hash))
->executeOne(); ->executeOne();
}
if (strlen($name) && !$hash && !$file) {
if ($length > PhabricatorFileStorageEngine::getChunkThreshold()) {
// If we don't have a hash, but this file is large enough to store in
// chunks and thus may be resumable, try to find a partially uploaded
// file by the same author with the same name and same length. This
// allows us to resume uploads in Javascript where we can't efficiently
// compute file hashes.
$file = id(new PhabricatorFileQuery())
->setViewer($viewer)
->withAuthorPHIDs(array($viewer->getPHID()))
->withNames(array($name))
->withLengthBetween($length, $length)
->withIsPartial(true)
->setLimit(1)
->executeOne();
}
}
if ($file) { if ($file) {
return array( return array(
@ -74,10 +89,8 @@ final class FileAllocateConduitAPIMethod
'filePHID' => $file->getPHID(), 'filePHID' => $file->getPHID(),
); );
} }
}
$engines = PhabricatorFileStorageEngine::loadStorageEngines( $engines = PhabricatorFileStorageEngine::loadStorageEngines($length);
$content_length);
if ($engines) { if ($engines) {
if ($force_chunking) { if ($force_chunking) {
@ -111,7 +124,7 @@ final class FileAllocateConduitAPIMethod
); );
} }
$file = $engine->allocateChunks($content_length, $chunk_properties); $file = $engine->allocateChunks($length, $chunk_properties);
return array( return array(
'upload' => true, 'upload' => true,

View file

@ -105,7 +105,16 @@ final class PhabricatorChunkedFileStorageEngine
} }
$input = $viewer->getAccountSecret().':'.$hash.':'.$viewer->getPHID(); $input = $viewer->getAccountSecret().':'.$hash.':'.$viewer->getPHID();
return PhabricatorHash::digest($input); return self::getChunkedHashForInput($input);
}
public static function getChunkedHashForInput($input) {
$rehash = PhabricatorHash::digest($input);
// Add a suffix to identify this as a chunk hash.
$rehash = substr($rehash, 0, -2).'-C';
return $rehash;
} }
public function allocateChunks($length, array $properties) { public function allocateChunks($length, array $properties) {

View file

@ -11,6 +11,10 @@ final class PhabricatorFileQuery
private $dateCreatedAfter; private $dateCreatedAfter;
private $dateCreatedBefore; private $dateCreatedBefore;
private $contentHashes; private $contentHashes;
private $minLength;
private $maxLength;
private $names;
private $isPartial;
public function withIDs(array $ids) { public function withIDs(array $ids) {
$this->ids = $ids; $this->ids = $ids;
@ -89,6 +93,22 @@ final class PhabricatorFileQuery
return $this; return $this;
} }
public function withLengthBetween($min, $max) {
$this->minLength = $min;
$this->maxLength = $max;
return $this;
}
public function withNames(array $names) {
$this->names = $names;
return $this;
}
public function withIsPartial($partial) {
$this->isPartial = $partial;
return $this;
}
public function showOnlyExplicitUploads($explicit_uploads) { public function showOnlyExplicitUploads($explicit_uploads) {
$this->explicitUploads = $explicit_uploads; $this->explicitUploads = $explicit_uploads;
return $this; return $this;
@ -213,34 +233,34 @@ final class PhabricatorFileQuery
$where[] = $this->buildPagingClause($conn_r); $where[] = $this->buildPagingClause($conn_r);
if ($this->ids) { if ($this->ids !== null) {
$where[] = qsprintf( $where[] = qsprintf(
$conn_r, $conn_r,
'f.id IN (%Ld)', 'f.id IN (%Ld)',
$this->ids); $this->ids);
} }
if ($this->phids) { if ($this->phids !== null) {
$where[] = qsprintf( $where[] = qsprintf(
$conn_r, $conn_r,
'f.phid IN (%Ls)', 'f.phid IN (%Ls)',
$this->phids); $this->phids);
} }
if ($this->authorPHIDs) { if ($this->authorPHIDs !== null) {
$where[] = qsprintf( $where[] = qsprintf(
$conn_r, $conn_r,
'f.authorPHID IN (%Ls)', 'f.authorPHID IN (%Ls)',
$this->authorPHIDs); $this->authorPHIDs);
} }
if ($this->explicitUploads) { if ($this->explicitUploads !== null) {
$where[] = qsprintf( $where[] = qsprintf(
$conn_r, $conn_r,
'f.isExplicitUpload = true'); 'f.isExplicitUpload = true');
} }
if ($this->transforms) { if ($this->transforms !== null) {
$clauses = array(); $clauses = array();
foreach ($this->transforms as $transform) { foreach ($this->transforms as $transform) {
if ($transform['transform'] === true) { if ($transform['transform'] === true) {
@ -259,27 +279,55 @@ final class PhabricatorFileQuery
$where[] = qsprintf($conn_r, '(%Q)', implode(') OR (', $clauses)); $where[] = qsprintf($conn_r, '(%Q)', implode(') OR (', $clauses));
} }
if ($this->dateCreatedAfter) { if ($this->dateCreatedAfter !== null) {
$where[] = qsprintf( $where[] = qsprintf(
$conn_r, $conn_r,
'f.dateCreated >= %d', 'f.dateCreated >= %d',
$this->dateCreatedAfter); $this->dateCreatedAfter);
} }
if ($this->dateCreatedBefore) { if ($this->dateCreatedBefore !== null) {
$where[] = qsprintf( $where[] = qsprintf(
$conn_r, $conn_r,
'f.dateCreated <= %d', 'f.dateCreated <= %d',
$this->dateCreatedBefore); $this->dateCreatedBefore);
} }
if ($this->contentHashes) { if ($this->contentHashes !== null) {
$where[] = qsprintf( $where[] = qsprintf(
$conn_r, $conn_r,
'f.contentHash IN (%Ls)', 'f.contentHash IN (%Ls)',
$this->contentHashes); $this->contentHashes);
} }
if ($this->minLength !== null) {
$where[] = qsprintf(
$conn_r,
'byteSize >= %d',
$this->minLength);
}
if ($this->maxLength !== null) {
$where[] = qsprintf(
$conn_r,
'byteSize <= %d',
$this->maxLength);
}
if ($this->names !== null) {
$where[] = qsprintf(
$conn_r,
'name in (%Ls)',
$this->names);
}
if ($this->isPartial !== null) {
$where[] = qsprintf(
$conn_r,
'isPartial = %d',
(int)$this->isPartial);
}
return $this->formatWhereClause($where); return $this->formatWhereClause($where);
} }

View file

@ -292,9 +292,10 @@ final class PhabricatorFile extends PhabricatorFileDAO
} else { } else {
// See PhabricatorChunkedFileStorageEngine::getChunkedHash() for some // See PhabricatorChunkedFileStorageEngine::getChunkedHash() for some
// discussion of this. // discussion of this.
$file->setContentHash( $seed = Filesystem::readRandomBytes(64);
PhabricatorHash::digest( $hash = PhabricatorChunkedFileStorageEngine::getChunkedHashForInput(
Filesystem::readRandomBytes(64))); $seed);
$file->setContentHash($hash);
} }
$file->setStorageEngine($engine->getEngineIdentifier()); $file->setStorageEngine($engine->getEngineIdentifier());