mirror of
https://we.phorge.it/source/phorge.git
synced 2024-11-23 15:22:41 +01:00
Queue large ICS files for background import
Summary: Ref T11801. When a file is larger than 512KB, queue it for background import instead of trying to do it in the foreground, sinc we risk hitting `max_execution_time`. Test Plan: {F1906943} Reviewers: chad Reviewed By: chad Maniphest Tasks: T11801 Differential Revision: https://secure.phabricator.com/D16805
This commit is contained in:
parent
e1566bef63
commit
17bd483207
8 changed files with 108 additions and 9 deletions
|
@ -2145,6 +2145,7 @@ phutil_register_library_map(array(
|
|||
'PhabricatorCalendarImportOrphanLogType' => 'applications/calendar/importlog/PhabricatorCalendarImportOrphanLogType.php',
|
||||
'PhabricatorCalendarImportPHIDType' => 'applications/calendar/phid/PhabricatorCalendarImportPHIDType.php',
|
||||
'PhabricatorCalendarImportQuery' => 'applications/calendar/query/PhabricatorCalendarImportQuery.php',
|
||||
'PhabricatorCalendarImportQueueLogType' => 'applications/calendar/importlog/PhabricatorCalendarImportQueueLogType.php',
|
||||
'PhabricatorCalendarImportReloadController' => 'applications/calendar/controller/PhabricatorCalendarImportReloadController.php',
|
||||
'PhabricatorCalendarImportReloadTransaction' => 'applications/calendar/xaction/PhabricatorCalendarImportReloadTransaction.php',
|
||||
'PhabricatorCalendarImportReloadWorker' => 'applications/calendar/worker/PhabricatorCalendarImportReloadWorker.php',
|
||||
|
@ -7012,6 +7013,7 @@ phutil_register_library_map(array(
|
|||
'PhabricatorCalendarImportOrphanLogType' => 'PhabricatorCalendarImportLogType',
|
||||
'PhabricatorCalendarImportPHIDType' => 'PhabricatorPHIDType',
|
||||
'PhabricatorCalendarImportQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
|
||||
'PhabricatorCalendarImportQueueLogType' => 'PhabricatorCalendarImportLogType',
|
||||
'PhabricatorCalendarImportReloadController' => 'PhabricatorCalendarController',
|
||||
'PhabricatorCalendarImportReloadTransaction' => 'PhabricatorCalendarImportTransactionType',
|
||||
'PhabricatorCalendarImportReloadWorker' => 'PhabricatorWorker',
|
||||
|
|
|
@ -54,7 +54,7 @@ final class PhabricatorCalendarImportEditor
|
|||
|
||||
if ($should_reload) {
|
||||
$import_engine = $object->getEngine();
|
||||
$import_engine->importEventsFromSource($actor, $object);
|
||||
$import_engine->importEventsFromSource($actor, $object, true);
|
||||
}
|
||||
|
||||
if ($should_trigger) {
|
||||
|
@ -107,6 +107,7 @@ final class PhabricatorCalendarImportEditor
|
|||
'class' => 'PhabricatorCalendarImportReloadWorker',
|
||||
'data' => array(
|
||||
'importPHID' => $object->getPHID(),
|
||||
'via' => PhabricatorCalendarImportReloadWorker::VIA_TRIGGER,
|
||||
),
|
||||
'options' => array(
|
||||
'objectPHID' => $object->getPHID(),
|
||||
|
|
|
@ -65,7 +65,8 @@ final class PhabricatorCalendarICSFileImportEngine
|
|||
|
||||
public function importEventsFromSource(
|
||||
PhabricatorUser $viewer,
|
||||
PhabricatorCalendarImport $import) {
|
||||
PhabricatorCalendarImport $import,
|
||||
$should_queue) {
|
||||
|
||||
$phid_key = PhabricatorCalendarImportICSFileTransaction::PARAMKEY_FILE;
|
||||
$file_phid = $import->getParameter($phid_key);
|
||||
|
@ -83,10 +84,13 @@ final class PhabricatorCalendarICSFileImportEngine
|
|||
|
||||
$data = $file->loadFileData();
|
||||
|
||||
if ($should_queue && $this->shouldQueueDataImport($data)) {
|
||||
return $this->queueDataImport($import, $data);
|
||||
}
|
||||
|
||||
return $this->importICSData($viewer, $import, $data);
|
||||
}
|
||||
|
||||
|
||||
public function canDisable(
|
||||
PhabricatorUser $viewer,
|
||||
PhabricatorCalendarImport $import) {
|
||||
|
|
|
@ -77,7 +77,8 @@ final class PhabricatorCalendarICSURIImportEngine
|
|||
|
||||
public function importEventsFromSource(
|
||||
PhabricatorUser $viewer,
|
||||
PhabricatorCalendarImport $import) {
|
||||
PhabricatorCalendarImport $import,
|
||||
$should_queue) {
|
||||
|
||||
$uri_key = PhabricatorCalendarImportICSURITransaction::PARAMKEY_URI;
|
||||
$uri = $import->getParameter($uri_key);
|
||||
|
@ -103,6 +104,10 @@ final class PhabricatorCalendarICSURIImportEngine
|
|||
|
||||
$data = $file->loadFileData();
|
||||
|
||||
if ($should_queue && $this->shouldQueueDataImport($data)) {
|
||||
return $this->queueDataImport($import, $data);
|
||||
}
|
||||
|
||||
return $this->importICSData($viewer, $import, $data);
|
||||
}
|
||||
|
||||
|
|
|
@ -3,11 +3,12 @@
|
|||
abstract class PhabricatorCalendarImportEngine
|
||||
extends Phobject {
|
||||
|
||||
const QUEUE_BYTE_LIMIT = 524288;
|
||||
|
||||
final public function getImportEngineType() {
|
||||
return $this->getPhobjectClassConstant('ENGINETYPE', 64);
|
||||
}
|
||||
|
||||
|
||||
abstract public function getImportEngineName();
|
||||
abstract public function getImportEngineTypeName();
|
||||
abstract public function getImportEngineHint();
|
||||
|
@ -27,7 +28,8 @@ abstract class PhabricatorCalendarImportEngine
|
|||
|
||||
abstract public function importEventsFromSource(
|
||||
PhabricatorUser $viewer,
|
||||
PhabricatorCalendarImport $import);
|
||||
PhabricatorCalendarImport $import,
|
||||
$should_queue);
|
||||
|
||||
abstract public function canDisable(
|
||||
PhabricatorUser $viewer,
|
||||
|
@ -568,4 +570,35 @@ abstract class PhabricatorCalendarImportEngine
|
|||
return (bool)$any_event;
|
||||
}
|
||||
|
||||
final protected function shouldQueueDataImport($data) {
|
||||
return (strlen($data) > self::QUEUE_BYTE_LIMIT);
|
||||
}
|
||||
|
||||
final protected function queueDataImport(
|
||||
PhabricatorCalendarImport $import,
|
||||
$data) {
|
||||
|
||||
$import->newLogMessage(
|
||||
PhabricatorCalendarImportQueueLogType::LOGTYPE,
|
||||
array(
|
||||
'data.size' => strlen($data),
|
||||
'data.limit' => self::QUEUE_BYTE_LIMIT,
|
||||
));
|
||||
|
||||
// When we queue on this pathway, we're queueing in response to an explicit
|
||||
// user action (like uploading a big `.ics` file), so we queue at normal
|
||||
// priority instead of bulk/import priority.
|
||||
|
||||
PhabricatorWorker::scheduleTask(
|
||||
'PhabricatorCalendarImportReloadWorker',
|
||||
array(
|
||||
'importPHID' => $import->getPHID(),
|
||||
'via' => PhabricatorCalendarImportReloadWorker::VIA_BACKGROUND,
|
||||
),
|
||||
array(
|
||||
'objectPHID' => $import->getPHID(),
|
||||
));
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -0,0 +1,40 @@
|
|||
<?php
|
||||
|
||||
final class PhabricatorCalendarImportQueueLogType
|
||||
extends PhabricatorCalendarImportLogType {
|
||||
|
||||
const LOGTYPE = 'queue';
|
||||
|
||||
public function getDisplayType(
|
||||
PhabricatorUser $viewer,
|
||||
PhabricatorCalendarImportLog $log) {
|
||||
return pht('Queued');
|
||||
}
|
||||
|
||||
public function getDisplayDescription(
|
||||
PhabricatorUser $viewer,
|
||||
PhabricatorCalendarImportLog $log) {
|
||||
|
||||
$size = $log->getParameter('data.size');
|
||||
$limit = $log->getParameter('data.limit');
|
||||
|
||||
return pht(
|
||||
'Queued for background import: data size (%s) exceeds limit for '.
|
||||
'immediate processing (%s).',
|
||||
phutil_format_bytes($size),
|
||||
phutil_format_bytes($limit));
|
||||
}
|
||||
|
||||
public function getDisplayIcon(
|
||||
PhabricatorUser $viewer,
|
||||
PhabricatorCalendarImportLog $log) {
|
||||
return 'fa-sort-amount-desc';
|
||||
}
|
||||
|
||||
public function getDisplayColor(
|
||||
PhabricatorUser $viewer,
|
||||
PhabricatorCalendarImportLog $log) {
|
||||
return 'blue';
|
||||
}
|
||||
|
||||
}
|
|
@ -14,7 +14,15 @@ final class PhabricatorCalendarImportTriggerLogType
|
|||
public function getDisplayDescription(
|
||||
PhabricatorUser $viewer,
|
||||
PhabricatorCalendarImportLog $log) {
|
||||
return pht('Triggered a periodic update.');
|
||||
|
||||
$via = $log->getParameter('via');
|
||||
switch ($via) {
|
||||
case PhabricatorCalendarImportReloadWorker::VIA_BACKGROUND:
|
||||
return pht('Started background processing.');
|
||||
case PhabricatorCalendarImportReloadWorker::VIA_TRIGGER:
|
||||
default:
|
||||
return pht('Triggered a periodic update.');
|
||||
}
|
||||
}
|
||||
|
||||
public function getDisplayIcon(
|
||||
|
|
|
@ -2,6 +2,9 @@
|
|||
|
||||
final class PhabricatorCalendarImportReloadWorker extends PhabricatorWorker {
|
||||
|
||||
const VIA_TRIGGER = 'trigger';
|
||||
const VIA_BACKGROUND = 'background';
|
||||
|
||||
protected function doWork() {
|
||||
$import = $this->loadImport();
|
||||
$viewer = PhabricatorUser::getOmnipotentUser();
|
||||
|
@ -18,11 +21,14 @@ final class PhabricatorCalendarImportReloadWorker extends PhabricatorWorker {
|
|||
|
||||
$import_engine = $import->getEngine();
|
||||
|
||||
$data = $this->getTaskData();
|
||||
$import->newLogMessage(
|
||||
PhabricatorCalendarImportTriggerLogType::LOGTYPE,
|
||||
array());
|
||||
array(
|
||||
'via' => idx($data, 'via', self::VIA_TRIGGER),
|
||||
));
|
||||
|
||||
$import_engine->importEventsFromSource($author, $import);
|
||||
$import_engine->importEventsFromSource($author, $import, false);
|
||||
}
|
||||
|
||||
private function loadImport() {
|
||||
|
|
Loading…
Reference in a new issue