mirror of
https://we.phorge.it/source/phorge.git
synced 2024-12-24 06:20:56 +01:00
Make logic for streaming VCS stuff directly to Files more reusable
Summary: Ref T11524. Ref T10423. Earlier, I converted `diffusion.filecontentquery` to put the actual file content in Files, then return a PHID for the file, instead of trying to send the content over Conduit. In T11524, we have a similar set of problems with diffs that contain non-UTF8 data (and, in T10423, diffs that are simply enormous). I want to provide an API method to do the same sort of thing with diff output (like from `git diff`), so we call the method, it shoves the data in Files, and then we go pull it out of Files. To support this, take the "shove the output of a Future into Files" logic and put it in a new base `FileFuture` query. This will let me make `RawDiffQuery` share the logic more easily. Test Plan: Browsed Diffusion, ran `diffusion.filecontentquery` to fetch file content. Reviewers: chad Reviewed By: chad Maniphest Tasks: T10423, T11524 Differential Revision: https://secure.phabricator.com/D16458
This commit is contained in:
parent
067d12d716
commit
771579496f
7 changed files with 161 additions and 143 deletions
|
@ -656,6 +656,7 @@ phutil_register_library_map(array(
|
||||||
'DiffusionExternalSymbolsSource' => 'applications/diffusion/symbol/DiffusionExternalSymbolsSource.php',
|
'DiffusionExternalSymbolsSource' => 'applications/diffusion/symbol/DiffusionExternalSymbolsSource.php',
|
||||||
'DiffusionFileContentQuery' => 'applications/diffusion/query/filecontent/DiffusionFileContentQuery.php',
|
'DiffusionFileContentQuery' => 'applications/diffusion/query/filecontent/DiffusionFileContentQuery.php',
|
||||||
'DiffusionFileContentQueryConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionFileContentQueryConduitAPIMethod.php',
|
'DiffusionFileContentQueryConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionFileContentQueryConduitAPIMethod.php',
|
||||||
|
'DiffusionFileFutureQuery' => 'applications/diffusion/query/DiffusionFileFutureQuery.php',
|
||||||
'DiffusionFindSymbolsConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionFindSymbolsConduitAPIMethod.php',
|
'DiffusionFindSymbolsConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionFindSymbolsConduitAPIMethod.php',
|
||||||
'DiffusionGetLintMessagesConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionGetLintMessagesConduitAPIMethod.php',
|
'DiffusionGetLintMessagesConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionGetLintMessagesConduitAPIMethod.php',
|
||||||
'DiffusionGetRecentCommitsByPathConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionGetRecentCommitsByPathConduitAPIMethod.php',
|
'DiffusionGetRecentCommitsByPathConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionGetRecentCommitsByPathConduitAPIMethod.php',
|
||||||
|
@ -5149,8 +5150,9 @@ phutil_register_library_map(array(
|
||||||
'DiffusionExternalController' => 'DiffusionController',
|
'DiffusionExternalController' => 'DiffusionController',
|
||||||
'DiffusionExternalSymbolQuery' => 'Phobject',
|
'DiffusionExternalSymbolQuery' => 'Phobject',
|
||||||
'DiffusionExternalSymbolsSource' => 'Phobject',
|
'DiffusionExternalSymbolsSource' => 'Phobject',
|
||||||
'DiffusionFileContentQuery' => 'DiffusionQuery',
|
'DiffusionFileContentQuery' => 'DiffusionFileFutureQuery',
|
||||||
'DiffusionFileContentQueryConduitAPIMethod' => 'DiffusionQueryConduitAPIMethod',
|
'DiffusionFileContentQueryConduitAPIMethod' => 'DiffusionQueryConduitAPIMethod',
|
||||||
|
'DiffusionFileFutureQuery' => 'DiffusionQuery',
|
||||||
'DiffusionFindSymbolsConduitAPIMethod' => 'DiffusionConduitAPIMethod',
|
'DiffusionFindSymbolsConduitAPIMethod' => 'DiffusionConduitAPIMethod',
|
||||||
'DiffusionGetLintMessagesConduitAPIMethod' => 'DiffusionConduitAPIMethod',
|
'DiffusionGetLintMessagesConduitAPIMethod' => 'DiffusionConduitAPIMethod',
|
||||||
'DiffusionGetRecentCommitsByPathConduitAPIMethod' => 'DiffusionConduitAPIMethod',
|
'DiffusionGetRecentCommitsByPathConduitAPIMethod' => 'DiffusionConduitAPIMethod',
|
||||||
|
|
|
@ -19,48 +19,14 @@ final class DiffusionFileContentQueryConduitAPIMethod
|
||||||
return array(
|
return array(
|
||||||
'path' => 'required string',
|
'path' => 'required string',
|
||||||
'commit' => 'required string',
|
'commit' => 'required string',
|
||||||
'timeout' => 'optional int',
|
) + DiffusionFileFutureQuery::getConduitParameters();
|
||||||
'byteLimit' => 'optional int',
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function getResult(ConduitAPIRequest $request) {
|
protected function getResult(ConduitAPIRequest $request) {
|
||||||
$drequest = $this->getDiffusionRequest();
|
$drequest = $this->getDiffusionRequest();
|
||||||
|
|
||||||
$file_query = DiffusionFileContentQuery::newFromDiffusionRequest($drequest);
|
return DiffusionFileContentQuery::newFromDiffusionRequest($drequest)
|
||||||
|
->respondToConduitRequest($request);
|
||||||
$timeout = $request->getValue('timeout');
|
|
||||||
if ($timeout) {
|
|
||||||
$file_query->setTimeout($timeout);
|
|
||||||
}
|
|
||||||
|
|
||||||
$byte_limit = $request->getValue('byteLimit');
|
|
||||||
if ($byte_limit) {
|
|
||||||
$file_query->setByteLimit($byte_limit);
|
|
||||||
}
|
|
||||||
|
|
||||||
$file = $file_query->execute();
|
|
||||||
|
|
||||||
$too_slow = (bool)$file_query->getExceededTimeLimit();
|
|
||||||
$too_huge = (bool)$file_query->getExceededByteLimit();
|
|
||||||
|
|
||||||
$file_phid = null;
|
|
||||||
if (!$too_slow && !$too_huge) {
|
|
||||||
$repository = $drequest->getRepository();
|
|
||||||
$repository_phid = $repository->getPHID();
|
|
||||||
|
|
||||||
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
|
|
||||||
$file->attachToObject($repository_phid);
|
|
||||||
unset($unguarded);
|
|
||||||
|
|
||||||
$file_phid = $file->getPHID();
|
|
||||||
}
|
|
||||||
|
|
||||||
return array(
|
|
||||||
'tooSlow' => $too_slow,
|
|
||||||
'tooHuge' => $too_huge,
|
|
||||||
'filePHID' => $file_phid,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
153
src/applications/diffusion/query/DiffusionFileFutureQuery.php
Normal file
153
src/applications/diffusion/query/DiffusionFileFutureQuery.php
Normal file
|
@ -0,0 +1,153 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
abstract class DiffusionFileFutureQuery
|
||||||
|
extends DiffusionQuery {
|
||||||
|
|
||||||
|
private $timeout;
|
||||||
|
private $byteLimit;
|
||||||
|
|
||||||
|
private $didHitByteLimit = false;
|
||||||
|
private $didHitTimeLimit = false;
|
||||||
|
|
||||||
|
public static function getConduitParameters() {
|
||||||
|
return array(
|
||||||
|
'timeout' => 'optional int',
|
||||||
|
'byteLimit' => 'optional int',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function setTimeout($timeout) {
|
||||||
|
$this->timeout = $timeout;
|
||||||
|
return $this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getTimeout() {
|
||||||
|
return $this->timeout;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function setByteLimit($byte_limit) {
|
||||||
|
$this->byteLimit = $byte_limit;
|
||||||
|
return $this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getByteLimit() {
|
||||||
|
return $this->byteLimit;
|
||||||
|
}
|
||||||
|
|
||||||
|
final public function getExceededByteLimit() {
|
||||||
|
return $this->didHitByteLimit;
|
||||||
|
}
|
||||||
|
|
||||||
|
final public function getExceededTimeLimit() {
|
||||||
|
return $this->didHitTimeLimit;
|
||||||
|
}
|
||||||
|
|
||||||
|
abstract protected function getFileContentFuture();
|
||||||
|
|
||||||
|
final public function respondToConduitRequest(ConduitAPIRequest $request) {
|
||||||
|
$drequest = $this->getRequest();
|
||||||
|
|
||||||
|
$timeout = $request->getValue('timeout');
|
||||||
|
if ($timeout) {
|
||||||
|
$this->setTimeout($timeout);
|
||||||
|
}
|
||||||
|
|
||||||
|
$byte_limit = $request->getValue('byteLimit');
|
||||||
|
if ($byte_limit) {
|
||||||
|
$this->setByteLimit($byte_limit);
|
||||||
|
}
|
||||||
|
|
||||||
|
$file = $this->execute();
|
||||||
|
|
||||||
|
$too_slow = (bool)$this->getExceededTimeLimit();
|
||||||
|
$too_huge = (bool)$this->getExceededByteLimit();
|
||||||
|
|
||||||
|
$file_phid = null;
|
||||||
|
if (!$too_slow && !$too_huge) {
|
||||||
|
$repository = $drequest->getRepository();
|
||||||
|
$repository_phid = $repository->getPHID();
|
||||||
|
|
||||||
|
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
|
||||||
|
$file->attachToObject($repository_phid);
|
||||||
|
unset($unguarded);
|
||||||
|
|
||||||
|
$file_phid = $file->getPHID();
|
||||||
|
}
|
||||||
|
|
||||||
|
return array(
|
||||||
|
'tooSlow' => $too_slow,
|
||||||
|
'tooHuge' => $too_huge,
|
||||||
|
'filePHID' => $file_phid,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
final public function executeInline() {
|
||||||
|
$future = $this->getFileContentFuture();
|
||||||
|
list($stdout) = $future->resolvex();
|
||||||
|
return $future;
|
||||||
|
}
|
||||||
|
|
||||||
|
final protected function executeQuery() {
|
||||||
|
$future = $this->newConfiguredFileContentFuture();
|
||||||
|
|
||||||
|
$drequest = $this->getRequest();
|
||||||
|
|
||||||
|
$name = basename($drequest->getPath());
|
||||||
|
$ttl = PhabricatorTime::getNow() + phutil_units('48 hours in seconds');
|
||||||
|
|
||||||
|
try {
|
||||||
|
$threshold = PhabricatorFileStorageEngine::getChunkThreshold();
|
||||||
|
$future->setReadBufferSize($threshold);
|
||||||
|
|
||||||
|
$source = id(new PhabricatorExecFutureFileUploadSource())
|
||||||
|
->setName($name)
|
||||||
|
->setTTL($ttl)
|
||||||
|
->setViewPolicy(PhabricatorPolicies::POLICY_NOONE)
|
||||||
|
->setExecFuture($future);
|
||||||
|
|
||||||
|
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
|
||||||
|
$file = $source->uploadFile();
|
||||||
|
unset($unguarded);
|
||||||
|
|
||||||
|
} catch (CommandException $ex) {
|
||||||
|
if (!$future->getWasKilledByTimeout()) {
|
||||||
|
throw $ex;
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->didHitTimeLimit = true;
|
||||||
|
$file = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
$byte_limit = $this->getByteLimit();
|
||||||
|
|
||||||
|
if ($byte_limit && ($file->getByteSize() > $byte_limit)) {
|
||||||
|
$this->didHitByteLimit = true;
|
||||||
|
|
||||||
|
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
|
||||||
|
id(new PhabricatorDestructionEngine())
|
||||||
|
->destroyObject($file);
|
||||||
|
unset($unguarded);
|
||||||
|
|
||||||
|
$file = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return $file;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function newConfiguredFileContentFuture() {
|
||||||
|
$future = $this->getFileContentFuture();
|
||||||
|
|
||||||
|
if ($this->getTimeout()) {
|
||||||
|
$future->setTimeout($this->getTimeout());
|
||||||
|
}
|
||||||
|
|
||||||
|
$byte_limit = $this->getByteLimit();
|
||||||
|
if ($byte_limit) {
|
||||||
|
$future->setStdoutSizeLimit($byte_limit + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
return $future;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
|
@ -1,99 +1,11 @@
|
||||||
<?php
|
<?php
|
||||||
|
|
||||||
abstract class DiffusionFileContentQuery extends DiffusionQuery {
|
abstract class DiffusionFileContentQuery
|
||||||
|
extends DiffusionFileFutureQuery {
|
||||||
private $timeout;
|
|
||||||
private $byteLimit;
|
|
||||||
|
|
||||||
private $didHitByteLimit = false;
|
|
||||||
private $didHitTimeLimit = false;
|
|
||||||
|
|
||||||
public function setTimeout($timeout) {
|
|
||||||
$this->timeout = $timeout;
|
|
||||||
return $this;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function getTimeout() {
|
|
||||||
return $this->timeout;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function setByteLimit($byte_limit) {
|
|
||||||
$this->byteLimit = $byte_limit;
|
|
||||||
return $this;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function getByteLimit() {
|
|
||||||
return $this->byteLimit;
|
|
||||||
}
|
|
||||||
|
|
||||||
final public static function newFromDiffusionRequest(
|
final public static function newFromDiffusionRequest(
|
||||||
DiffusionRequest $request) {
|
DiffusionRequest $request) {
|
||||||
return parent::newQueryObject(__CLASS__, $request);
|
return parent::newQueryObject(__CLASS__, $request);
|
||||||
}
|
}
|
||||||
|
|
||||||
final public function getExceededByteLimit() {
|
|
||||||
return $this->didHitByteLimit;
|
|
||||||
}
|
|
||||||
|
|
||||||
final public function getExceededTimeLimit() {
|
|
||||||
return $this->didHitTimeLimit;
|
|
||||||
}
|
|
||||||
|
|
||||||
abstract protected function getFileContentFuture();
|
|
||||||
abstract protected function resolveFileContentFuture(Future $future);
|
|
||||||
|
|
||||||
final protected function executeQuery() {
|
|
||||||
$future = $this->getFileContentFuture();
|
|
||||||
|
|
||||||
if ($this->getTimeout()) {
|
|
||||||
$future->setTimeout($this->getTimeout());
|
|
||||||
}
|
|
||||||
|
|
||||||
$byte_limit = $this->getByteLimit();
|
|
||||||
if ($byte_limit) {
|
|
||||||
$future->setStdoutSizeLimit($byte_limit + 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
$drequest = $this->getRequest();
|
|
||||||
|
|
||||||
$name = basename($drequest->getPath());
|
|
||||||
$ttl = PhabricatorTime::getNow() + phutil_units('48 hours in seconds');
|
|
||||||
|
|
||||||
try {
|
|
||||||
$threshold = PhabricatorFileStorageEngine::getChunkThreshold();
|
|
||||||
$future->setReadBufferSize($threshold);
|
|
||||||
|
|
||||||
$source = id(new PhabricatorExecFutureFileUploadSource())
|
|
||||||
->setName($name)
|
|
||||||
->setTTL($ttl)
|
|
||||||
->setViewPolicy(PhabricatorPolicies::POLICY_NOONE)
|
|
||||||
->setExecFuture($future);
|
|
||||||
|
|
||||||
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
|
|
||||||
$file = $source->uploadFile();
|
|
||||||
unset($unguarded);
|
|
||||||
|
|
||||||
} catch (CommandException $ex) {
|
|
||||||
if (!$future->getWasKilledByTimeout()) {
|
|
||||||
throw $ex;
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->didHitTimeLimit = true;
|
|
||||||
$file = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($byte_limit && ($file->getByteSize() > $byte_limit)) {
|
|
||||||
$this->didHitByteLimit = true;
|
|
||||||
|
|
||||||
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
|
|
||||||
id(new PhabricatorDestructionEngine())
|
|
||||||
->destroyObject($file);
|
|
||||||
unset($unguarded);
|
|
||||||
|
|
||||||
$file = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return $file;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,9 +15,4 @@ final class DiffusionGitFileContentQuery extends DiffusionFileContentQuery {
|
||||||
$path);
|
$path);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function resolveFileContentFuture(Future $future) {
|
|
||||||
list($corpus) = $future->resolvex();
|
|
||||||
return $corpus;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,9 +16,4 @@ final class DiffusionMercurialFileContentQuery
|
||||||
$path);
|
$path);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function resolveFileContentFuture(Future $future) {
|
|
||||||
list($corpus) = $future->resolvex();
|
|
||||||
return $corpus;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,9 +14,4 @@ final class DiffusionSvnFileContentQuery extends DiffusionFileContentQuery {
|
||||||
$repository->getSubversionPathURI($path, $commit));
|
$repository->getSubversionPathURI($path, $commit));
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function resolveFileContentFuture(Future $future) {
|
|
||||||
list($corpus) = $future->resolvex();
|
|
||||||
return $corpus;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue