mirror of
https://we.phorge.it/source/phorge.git
synced 2024-12-24 06:20:56 +01:00
Make logic for streaming VCS stuff directly to Files more reusable
Summary: Ref T11524. Ref T10423. Earlier, I converted `diffusion.filecontentquery` to put the actual file content in Files, then return a PHID for the file, instead of trying to send the content over Conduit. In T11524, we have a similar set of problems with diffs that contain non-UTF8 data (and, in T10423, diffs that are simply enormous). I want to provide an API method to do the same sort of thing with diff output (like from `git diff`), so we call the method, it shoves the data in Files, and then we go pull it out of Files. To support this, take the "shove the output of a Future into Files" logic and put it in a new base `FileFuture` query. This will let me make `RawDiffQuery` share the logic more easily. Test Plan: Browsed Diffusion, ran `diffusion.filecontentquery` to fetch file content. Reviewers: chad Reviewed By: chad Maniphest Tasks: T10423, T11524 Differential Revision: https://secure.phabricator.com/D16458
This commit is contained in:
parent
067d12d716
commit
771579496f
7 changed files with 161 additions and 143 deletions
|
@ -656,6 +656,7 @@ phutil_register_library_map(array(
|
|||
'DiffusionExternalSymbolsSource' => 'applications/diffusion/symbol/DiffusionExternalSymbolsSource.php',
|
||||
'DiffusionFileContentQuery' => 'applications/diffusion/query/filecontent/DiffusionFileContentQuery.php',
|
||||
'DiffusionFileContentQueryConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionFileContentQueryConduitAPIMethod.php',
|
||||
'DiffusionFileFutureQuery' => 'applications/diffusion/query/DiffusionFileFutureQuery.php',
|
||||
'DiffusionFindSymbolsConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionFindSymbolsConduitAPIMethod.php',
|
||||
'DiffusionGetLintMessagesConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionGetLintMessagesConduitAPIMethod.php',
|
||||
'DiffusionGetRecentCommitsByPathConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionGetRecentCommitsByPathConduitAPIMethod.php',
|
||||
|
@ -5149,8 +5150,9 @@ phutil_register_library_map(array(
|
|||
'DiffusionExternalController' => 'DiffusionController',
|
||||
'DiffusionExternalSymbolQuery' => 'Phobject',
|
||||
'DiffusionExternalSymbolsSource' => 'Phobject',
|
||||
'DiffusionFileContentQuery' => 'DiffusionQuery',
|
||||
'DiffusionFileContentQuery' => 'DiffusionFileFutureQuery',
|
||||
'DiffusionFileContentQueryConduitAPIMethod' => 'DiffusionQueryConduitAPIMethod',
|
||||
'DiffusionFileFutureQuery' => 'DiffusionQuery',
|
||||
'DiffusionFindSymbolsConduitAPIMethod' => 'DiffusionConduitAPIMethod',
|
||||
'DiffusionGetLintMessagesConduitAPIMethod' => 'DiffusionConduitAPIMethod',
|
||||
'DiffusionGetRecentCommitsByPathConduitAPIMethod' => 'DiffusionConduitAPIMethod',
|
||||
|
|
|
@ -19,48 +19,14 @@ final class DiffusionFileContentQueryConduitAPIMethod
|
|||
return array(
|
||||
'path' => 'required string',
|
||||
'commit' => 'required string',
|
||||
'timeout' => 'optional int',
|
||||
'byteLimit' => 'optional int',
|
||||
);
|
||||
) + DiffusionFileFutureQuery::getConduitParameters();
|
||||
}
|
||||
|
||||
protected function getResult(ConduitAPIRequest $request) {
|
||||
$drequest = $this->getDiffusionRequest();
|
||||
|
||||
$file_query = DiffusionFileContentQuery::newFromDiffusionRequest($drequest);
|
||||
|
||||
$timeout = $request->getValue('timeout');
|
||||
if ($timeout) {
|
||||
$file_query->setTimeout($timeout);
|
||||
}
|
||||
|
||||
$byte_limit = $request->getValue('byteLimit');
|
||||
if ($byte_limit) {
|
||||
$file_query->setByteLimit($byte_limit);
|
||||
}
|
||||
|
||||
$file = $file_query->execute();
|
||||
|
||||
$too_slow = (bool)$file_query->getExceededTimeLimit();
|
||||
$too_huge = (bool)$file_query->getExceededByteLimit();
|
||||
|
||||
$file_phid = null;
|
||||
if (!$too_slow && !$too_huge) {
|
||||
$repository = $drequest->getRepository();
|
||||
$repository_phid = $repository->getPHID();
|
||||
|
||||
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
|
||||
$file->attachToObject($repository_phid);
|
||||
unset($unguarded);
|
||||
|
||||
$file_phid = $file->getPHID();
|
||||
}
|
||||
|
||||
return array(
|
||||
'tooSlow' => $too_slow,
|
||||
'tooHuge' => $too_huge,
|
||||
'filePHID' => $file_phid,
|
||||
);
|
||||
return DiffusionFileContentQuery::newFromDiffusionRequest($drequest)
|
||||
->respondToConduitRequest($request);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
153
src/applications/diffusion/query/DiffusionFileFutureQuery.php
Normal file
153
src/applications/diffusion/query/DiffusionFileFutureQuery.php
Normal file
|
@ -0,0 +1,153 @@
|
|||
<?php
|
||||
|
||||
abstract class DiffusionFileFutureQuery
|
||||
extends DiffusionQuery {
|
||||
|
||||
private $timeout;
|
||||
private $byteLimit;
|
||||
|
||||
private $didHitByteLimit = false;
|
||||
private $didHitTimeLimit = false;
|
||||
|
||||
public static function getConduitParameters() {
|
||||
return array(
|
||||
'timeout' => 'optional int',
|
||||
'byteLimit' => 'optional int',
|
||||
);
|
||||
}
|
||||
|
||||
public function setTimeout($timeout) {
|
||||
$this->timeout = $timeout;
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getTimeout() {
|
||||
return $this->timeout;
|
||||
}
|
||||
|
||||
public function setByteLimit($byte_limit) {
|
||||
$this->byteLimit = $byte_limit;
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getByteLimit() {
|
||||
return $this->byteLimit;
|
||||
}
|
||||
|
||||
final public function getExceededByteLimit() {
|
||||
return $this->didHitByteLimit;
|
||||
}
|
||||
|
||||
final public function getExceededTimeLimit() {
|
||||
return $this->didHitTimeLimit;
|
||||
}
|
||||
|
||||
abstract protected function getFileContentFuture();
|
||||
|
||||
final public function respondToConduitRequest(ConduitAPIRequest $request) {
|
||||
$drequest = $this->getRequest();
|
||||
|
||||
$timeout = $request->getValue('timeout');
|
||||
if ($timeout) {
|
||||
$this->setTimeout($timeout);
|
||||
}
|
||||
|
||||
$byte_limit = $request->getValue('byteLimit');
|
||||
if ($byte_limit) {
|
||||
$this->setByteLimit($byte_limit);
|
||||
}
|
||||
|
||||
$file = $this->execute();
|
||||
|
||||
$too_slow = (bool)$this->getExceededTimeLimit();
|
||||
$too_huge = (bool)$this->getExceededByteLimit();
|
||||
|
||||
$file_phid = null;
|
||||
if (!$too_slow && !$too_huge) {
|
||||
$repository = $drequest->getRepository();
|
||||
$repository_phid = $repository->getPHID();
|
||||
|
||||
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
|
||||
$file->attachToObject($repository_phid);
|
||||
unset($unguarded);
|
||||
|
||||
$file_phid = $file->getPHID();
|
||||
}
|
||||
|
||||
return array(
|
||||
'tooSlow' => $too_slow,
|
||||
'tooHuge' => $too_huge,
|
||||
'filePHID' => $file_phid,
|
||||
);
|
||||
}
|
||||
|
||||
final public function executeInline() {
|
||||
$future = $this->getFileContentFuture();
|
||||
list($stdout) = $future->resolvex();
|
||||
return $future;
|
||||
}
|
||||
|
||||
final protected function executeQuery() {
|
||||
$future = $this->newConfiguredFileContentFuture();
|
||||
|
||||
$drequest = $this->getRequest();
|
||||
|
||||
$name = basename($drequest->getPath());
|
||||
$ttl = PhabricatorTime::getNow() + phutil_units('48 hours in seconds');
|
||||
|
||||
try {
|
||||
$threshold = PhabricatorFileStorageEngine::getChunkThreshold();
|
||||
$future->setReadBufferSize($threshold);
|
||||
|
||||
$source = id(new PhabricatorExecFutureFileUploadSource())
|
||||
->setName($name)
|
||||
->setTTL($ttl)
|
||||
->setViewPolicy(PhabricatorPolicies::POLICY_NOONE)
|
||||
->setExecFuture($future);
|
||||
|
||||
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
|
||||
$file = $source->uploadFile();
|
||||
unset($unguarded);
|
||||
|
||||
} catch (CommandException $ex) {
|
||||
if (!$future->getWasKilledByTimeout()) {
|
||||
throw $ex;
|
||||
}
|
||||
|
||||
$this->didHitTimeLimit = true;
|
||||
$file = null;
|
||||
}
|
||||
|
||||
$byte_limit = $this->getByteLimit();
|
||||
|
||||
if ($byte_limit && ($file->getByteSize() > $byte_limit)) {
|
||||
$this->didHitByteLimit = true;
|
||||
|
||||
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
|
||||
id(new PhabricatorDestructionEngine())
|
||||
->destroyObject($file);
|
||||
unset($unguarded);
|
||||
|
||||
$file = null;
|
||||
}
|
||||
|
||||
return $file;
|
||||
}
|
||||
|
||||
private function newConfiguredFileContentFuture() {
|
||||
$future = $this->getFileContentFuture();
|
||||
|
||||
if ($this->getTimeout()) {
|
||||
$future->setTimeout($this->getTimeout());
|
||||
}
|
||||
|
||||
$byte_limit = $this->getByteLimit();
|
||||
if ($byte_limit) {
|
||||
$future->setStdoutSizeLimit($byte_limit + 1);
|
||||
}
|
||||
|
||||
return $future;
|
||||
}
|
||||
|
||||
|
||||
}
|
|
@ -1,99 +1,11 @@
|
|||
<?php
|
||||
|
||||
abstract class DiffusionFileContentQuery extends DiffusionQuery {
|
||||
|
||||
private $timeout;
|
||||
private $byteLimit;
|
||||
|
||||
private $didHitByteLimit = false;
|
||||
private $didHitTimeLimit = false;
|
||||
|
||||
public function setTimeout($timeout) {
|
||||
$this->timeout = $timeout;
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getTimeout() {
|
||||
return $this->timeout;
|
||||
}
|
||||
|
||||
public function setByteLimit($byte_limit) {
|
||||
$this->byteLimit = $byte_limit;
|
||||
return $this;
|
||||
}
|
||||
|
||||
public function getByteLimit() {
|
||||
return $this->byteLimit;
|
||||
}
|
||||
abstract class DiffusionFileContentQuery
|
||||
extends DiffusionFileFutureQuery {
|
||||
|
||||
final public static function newFromDiffusionRequest(
|
||||
DiffusionRequest $request) {
|
||||
return parent::newQueryObject(__CLASS__, $request);
|
||||
}
|
||||
|
||||
final public function getExceededByteLimit() {
|
||||
return $this->didHitByteLimit;
|
||||
}
|
||||
|
||||
final public function getExceededTimeLimit() {
|
||||
return $this->didHitTimeLimit;
|
||||
}
|
||||
|
||||
abstract protected function getFileContentFuture();
|
||||
abstract protected function resolveFileContentFuture(Future $future);
|
||||
|
||||
final protected function executeQuery() {
|
||||
$future = $this->getFileContentFuture();
|
||||
|
||||
if ($this->getTimeout()) {
|
||||
$future->setTimeout($this->getTimeout());
|
||||
}
|
||||
|
||||
$byte_limit = $this->getByteLimit();
|
||||
if ($byte_limit) {
|
||||
$future->setStdoutSizeLimit($byte_limit + 1);
|
||||
}
|
||||
|
||||
$drequest = $this->getRequest();
|
||||
|
||||
$name = basename($drequest->getPath());
|
||||
$ttl = PhabricatorTime::getNow() + phutil_units('48 hours in seconds');
|
||||
|
||||
try {
|
||||
$threshold = PhabricatorFileStorageEngine::getChunkThreshold();
|
||||
$future->setReadBufferSize($threshold);
|
||||
|
||||
$source = id(new PhabricatorExecFutureFileUploadSource())
|
||||
->setName($name)
|
||||
->setTTL($ttl)
|
||||
->setViewPolicy(PhabricatorPolicies::POLICY_NOONE)
|
||||
->setExecFuture($future);
|
||||
|
||||
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
|
||||
$file = $source->uploadFile();
|
||||
unset($unguarded);
|
||||
|
||||
} catch (CommandException $ex) {
|
||||
if (!$future->getWasKilledByTimeout()) {
|
||||
throw $ex;
|
||||
}
|
||||
|
||||
$this->didHitTimeLimit = true;
|
||||
$file = null;
|
||||
}
|
||||
|
||||
if ($byte_limit && ($file->getByteSize() > $byte_limit)) {
|
||||
$this->didHitByteLimit = true;
|
||||
|
||||
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
|
||||
id(new PhabricatorDestructionEngine())
|
||||
->destroyObject($file);
|
||||
unset($unguarded);
|
||||
|
||||
$file = null;
|
||||
}
|
||||
|
||||
return $file;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -15,9 +15,4 @@ final class DiffusionGitFileContentQuery extends DiffusionFileContentQuery {
|
|||
$path);
|
||||
}
|
||||
|
||||
protected function resolveFileContentFuture(Future $future) {
|
||||
list($corpus) = $future->resolvex();
|
||||
return $corpus;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -16,9 +16,4 @@ final class DiffusionMercurialFileContentQuery
|
|||
$path);
|
||||
}
|
||||
|
||||
protected function resolveFileContentFuture(Future $future) {
|
||||
list($corpus) = $future->resolvex();
|
||||
return $corpus;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -14,9 +14,4 @@ final class DiffusionSvnFileContentQuery extends DiffusionFileContentQuery {
|
|||
$repository->getSubversionPathURI($path, $commit));
|
||||
}
|
||||
|
||||
protected function resolveFileContentFuture(Future $future) {
|
||||
list($corpus) = $future->resolvex();
|
||||
return $corpus;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue