1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-22 14:52:41 +01:00

Complete modularization of the GC daemon

Summary: This modularizes the rest of the GC submethods. Turned out there was nothing tricky.

Test Plan: Ran `bin/phd debug garbage` and got reasonable looking behavior and output.

Reviewers: btrahan

Reviewed By: btrahan

CC: aran

Differential Revision: https://secure.phabricator.com/D7971
This commit is contained in:
epriestley 2014-01-15 10:02:31 -08:00
parent 56d44f1503
commit 9f35c7cc26
11 changed files with 259 additions and 234 deletions

View file

@ -259,7 +259,9 @@ phutil_register_library_map(array(
'ConduitAPI_user_whoami_Method' => 'applications/people/conduit/ConduitAPI_user_whoami_Method.php',
'ConduitCall' => 'applications/conduit/call/ConduitCall.php',
'ConduitCallTestCase' => 'applications/conduit/call/__tests__/ConduitCallTestCase.php',
'ConduitConnectionGarbageCollector' => 'applications/conduit/garbagecollector/ConduitConnectionGarbageCollector.php',
'ConduitException' => 'applications/conduit/protocol/ConduitException.php',
'ConduitLogGarbageCollector' => 'applications/conduit/garbagecollector/ConduitLogGarbageCollector.php',
'ConduitSSHWorkflow' => 'applications/conduit/ssh/ConduitSSHWorkflow.php',
'ConpherenceActionMenuEventListener' => 'applications/conpherence/events/ConpherenceActionMenuEventListener.php',
'ConpherenceConfigOptions' => 'applications/conpherence/config/ConpherenceConfigOptions.php',
@ -413,6 +415,7 @@ phutil_register_library_map(array(
'DifferentialNewDiffMail' => 'applications/differential/mail/DifferentialNewDiffMail.php',
'DifferentialPHIDTypeDiff' => 'applications/differential/phid/DifferentialPHIDTypeDiff.php',
'DifferentialPHIDTypeRevision' => 'applications/differential/phid/DifferentialPHIDTypeRevision.php',
'DifferentialParseCacheGarbageCollector' => 'applications/differential/garbagecollector/DifferentialParseCacheGarbageCollector.php',
'DifferentialParseRenderTestCase' => 'applications/differential/__tests__/DifferentialParseRenderTestCase.php',
'DifferentialPathFieldSpecification' => 'applications/differential/field/specification/DifferentialPathFieldSpecification.php',
'DifferentialPrimaryPaneView' => 'applications/differential/view/DifferentialPrimaryPaneView.php',
@ -1236,8 +1239,11 @@ phutil_register_library_map(array(
'PhabricatorBuiltinPatchList' => 'infrastructure/storage/patch/PhabricatorBuiltinPatchList.php',
'PhabricatorBusyExample' => 'applications/uiexample/examples/PhabricatorBusyExample.php',
'PhabricatorCacheDAO' => 'applications/cache/storage/PhabricatorCacheDAO.php',
'PhabricatorCacheGeneralGarbageCollector' => 'applications/cache/garbagecollector/PhabricatorCacheGeneralGarbageCollector.php',
'PhabricatorCacheManagementPurgeWorkflow' => 'applications/cache/management/PhabricatorCacheManagementPurgeWorkflow.php',
'PhabricatorCacheManagementWorkflow' => 'applications/cache/management/PhabricatorCacheManagementWorkflow.php',
'PhabricatorCacheMarkupGarbageCollector' => 'applications/cache/garbagecollector/PhabricatorCacheMarkupGarbageCollector.php',
'PhabricatorCacheTTLGarbageCollector' => 'applications/cache/garbagecollector/PhabricatorCacheTTLGarbageCollector.php',
'PhabricatorCaches' => 'applications/cache/PhabricatorCaches.php',
'PhabricatorCalendarBrowseController' => 'applications/calendar/controller/PhabricatorCalendarBrowseController.php',
'PhabricatorCalendarController' => 'applications/calendar/controller/PhabricatorCalendarController.php',
@ -1350,6 +1356,7 @@ phutil_register_library_map(array(
'PhabricatorDaemonLogEvent' => 'applications/daemon/storage/PhabricatorDaemonLogEvent.php',
'PhabricatorDaemonLogEventViewController' => 'applications/daemon/controller/PhabricatorDaemonLogEventViewController.php',
'PhabricatorDaemonLogEventsView' => 'applications/daemon/view/PhabricatorDaemonLogEventsView.php',
'PhabricatorDaemonLogGarbageCollector' => 'applications/daemon/garbagecollector/PhabricatorDaemonLogGarbageCollector.php',
'PhabricatorDaemonLogListController' => 'applications/daemon/controller/PhabricatorDaemonLogListController.php',
'PhabricatorDaemonLogListView' => 'applications/daemon/view/PhabricatorDaemonLogListView.php',
'PhabricatorDaemonLogQuery' => 'applications/daemon/query/PhabricatorDaemonLogQuery.php',
@ -1364,6 +1371,7 @@ phutil_register_library_map(array(
'PhabricatorDaemonManagementStopWorkflow' => 'applications/daemon/management/PhabricatorDaemonManagementStopWorkflow.php',
'PhabricatorDaemonManagementWorkflow' => 'applications/daemon/management/PhabricatorDaemonManagementWorkflow.php',
'PhabricatorDaemonReference' => 'infrastructure/daemon/control/PhabricatorDaemonReference.php',
'PhabricatorDaemonTaskGarbageCollector' => 'applications/daemon/garbagecollector/PhabricatorDaemonTaskGarbageCollector.php',
'PhabricatorDataNotAttachedException' => 'infrastructure/storage/lisk/PhabricatorDataNotAttachedException.php',
'PhabricatorDebugController' => 'applications/system/PhabricatorDebugController.php',
'PhabricatorDefaultFileStorageEngineSelector' => 'applications/files/engineselector/PhabricatorDefaultFileStorageEngineSelector.php',
@ -1472,6 +1480,7 @@ phutil_register_library_map(array(
'PhabricatorFileStorageConfigurationException' => 'applications/files/exception/PhabricatorFileStorageConfigurationException.php',
'PhabricatorFileStorageEngine' => 'applications/files/engine/PhabricatorFileStorageEngine.php',
'PhabricatorFileStorageEngineSelector' => 'applications/files/engineselector/PhabricatorFileStorageEngineSelector.php',
'PhabricatorFileTemporaryGarbageCollector' => 'applications/files/garbagecollector/PhabricatorFileTemporaryGarbageCollector.php',
'PhabricatorFileTestCase' => 'applications/files/storage/__tests__/PhabricatorFileTestCase.php',
'PhabricatorFileTestDataGenerator' => 'applications/files/lipsum/PhabricatorFileTestDataGenerator.php',
'PhabricatorFileTransaction' => 'applications/files/storage/PhabricatorFileTransaction.php',
@ -2699,7 +2708,9 @@ phutil_register_library_map(array(
'ConduitAPI_user_removestatus_Method' => 'ConduitAPI_user_Method',
'ConduitAPI_user_whoami_Method' => 'ConduitAPI_user_Method',
'ConduitCallTestCase' => 'PhabricatorTestCase',
'ConduitConnectionGarbageCollector' => 'PhabricatorGarbageCollector',
'ConduitException' => 'Exception',
'ConduitLogGarbageCollector' => 'PhabricatorGarbageCollector',
'ConduitSSHWorkflow' => 'PhabricatorSSHWorkflow',
'ConpherenceActionMenuEventListener' => 'PhabricatorEventListener',
'ConpherenceConfigOptions' => 'PhabricatorApplicationConfigOptions',
@ -2847,6 +2858,7 @@ phutil_register_library_map(array(
'DifferentialNewDiffMail' => 'DifferentialReviewRequestMail',
'DifferentialPHIDTypeDiff' => 'PhabricatorPHIDType',
'DifferentialPHIDTypeRevision' => 'PhabricatorPHIDType',
'DifferentialParseCacheGarbageCollector' => 'PhabricatorGarbageCollector',
'DifferentialParseRenderTestCase' => 'PhabricatorTestCase',
'DifferentialPathFieldSpecification' => 'DifferentialFieldSpecification',
'DifferentialPrimaryPaneView' => 'AphrontView',
@ -3803,8 +3815,11 @@ phutil_register_library_map(array(
'PhabricatorBuiltinPatchList' => 'PhabricatorSQLPatchList',
'PhabricatorBusyExample' => 'PhabricatorUIExample',
'PhabricatorCacheDAO' => 'PhabricatorLiskDAO',
'PhabricatorCacheGeneralGarbageCollector' => 'PhabricatorGarbageCollector',
'PhabricatorCacheManagementPurgeWorkflow' => 'PhabricatorCacheManagementWorkflow',
'PhabricatorCacheManagementWorkflow' => 'PhabricatorManagementWorkflow',
'PhabricatorCacheMarkupGarbageCollector' => 'PhabricatorGarbageCollector',
'PhabricatorCacheTTLGarbageCollector' => 'PhabricatorGarbageCollector',
'PhabricatorCalendarBrowseController' => 'PhabricatorCalendarController',
'PhabricatorCalendarController' => 'PhabricatorController',
'PhabricatorCalendarDAO' => 'PhabricatorLiskDAO',
@ -3944,6 +3959,7 @@ phutil_register_library_map(array(
'PhabricatorDaemonLogEvent' => 'PhabricatorDaemonDAO',
'PhabricatorDaemonLogEventViewController' => 'PhabricatorDaemonController',
'PhabricatorDaemonLogEventsView' => 'AphrontView',
'PhabricatorDaemonLogGarbageCollector' => 'PhabricatorGarbageCollector',
'PhabricatorDaemonLogListController' => 'PhabricatorDaemonController',
'PhabricatorDaemonLogListView' => 'AphrontView',
'PhabricatorDaemonLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
@ -3957,6 +3973,7 @@ phutil_register_library_map(array(
'PhabricatorDaemonManagementStatusWorkflow' => 'PhabricatorDaemonManagementWorkflow',
'PhabricatorDaemonManagementStopWorkflow' => 'PhabricatorDaemonManagementWorkflow',
'PhabricatorDaemonManagementWorkflow' => 'PhabricatorManagementWorkflow',
'PhabricatorDaemonTaskGarbageCollector' => 'PhabricatorGarbageCollector',
'PhabricatorDataNotAttachedException' => 'Exception',
'PhabricatorDebugController' => 'PhabricatorController',
'PhabricatorDefaultFileStorageEngineSelector' => 'PhabricatorFileStorageEngineSelector',
@ -4079,6 +4096,7 @@ phutil_register_library_map(array(
'PhabricatorFileShortcutController' => 'PhabricatorFileController',
'PhabricatorFileStorageBlob' => 'PhabricatorFileDAO',
'PhabricatorFileStorageConfigurationException' => 'Exception',
'PhabricatorFileTemporaryGarbageCollector' => 'PhabricatorGarbageCollector',
'PhabricatorFileTestCase' => 'PhabricatorTestCase',
'PhabricatorFileTestDataGenerator' => 'PhabricatorTestDataGenerator',
'PhabricatorFileTransaction' => 'PhabricatorApplicationTransaction',

View file

@ -0,0 +1,26 @@
<?php
final class PhabricatorCacheGeneralGarbageCollector
extends PhabricatorGarbageCollector {
public function collectGarbage() {
$key = 'gcdaemon.ttl.general-cache';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return false;
}
$cache = new PhabricatorKeyValueDatabaseCache();
$conn_w = $cache->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE cacheCreated < %d
ORDER BY cacheCreated ASC LIMIT 100',
$cache->getTableName(),
time() - $ttl);
return ($conn_w->getAffectedRows() == 100);
}
}

View file

@ -0,0 +1,25 @@
<?php
final class PhabricatorCacheMarkupGarbageCollector
extends PhabricatorGarbageCollector {
public function collectGarbage() {
$key = 'gcdaemon.ttl.markup-cache';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return false;
}
$table = new PhabricatorMarkupCache();
$conn_w = $table->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE dateCreated < %d LIMIT 100',
$table->getTableName(),
time() - $ttl);
return ($conn_w->getAffectedRows() == 100);
}
}

View file

@ -0,0 +1,20 @@
<?php
final class PhabricatorCacheTTLGarbageCollector
extends PhabricatorGarbageCollector {
public function collectGarbage() {
$cache = new PhabricatorKeyValueDatabaseCache();
$conn_w = $cache->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE cacheExpires < %d
ORDER BY cacheExpires ASC LIMIT 100',
$cache->getTableName(),
time());
return ($conn_w->getAffectedRows() == 100);
}
}

View file

@ -0,0 +1,25 @@
<?php
final class ConduitConnectionGarbageCollector
extends PhabricatorGarbageCollector {
public function collectGarbage() {
$key = 'gcdaemon.ttl.conduit-logs';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return false;
}
$table = new PhabricatorConduitConnectionLog();
$conn_w = $table->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE dateCreated < %d
ORDER BY dateCreated ASC LIMIT 100',
$table->getTableName(),
time() - $ttl);
return ($conn_w->getAffectedRows() == 100);
}
}

View file

@ -0,0 +1,25 @@
<?php
final class ConduitLogGarbageCollector
extends PhabricatorGarbageCollector {
public function collectGarbage() {
$key = 'gcdaemon.ttl.conduit-logs';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return false;
}
$table = new PhabricatorConduitMethodCallLog();
$conn_w = $table->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE dateCreated < %d
ORDER BY dateCreated ASC LIMIT 100',
$table->getTableName(),
time() - $ttl);
return ($conn_w->getAffectedRows() == 100);
}
}

View file

@ -0,0 +1,24 @@
<?php
final class PhabricatorDaemonLogGarbageCollector
extends PhabricatorGarbageCollector {
public function collectGarbage() {
$ttl = PhabricatorEnv::getEnvConfig('gcdaemon.ttl.daemon-logs');
if ($ttl <= 0) {
return false;
}
$table = new PhabricatorDaemonLogEvent();
$conn_w = $table->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE epoch < %d LIMIT 100',
$table->getTableName(),
time() - $ttl);
return ($conn_w->getAffectedRows() == 100);
}
}

View file

@ -0,0 +1,48 @@
<?php
final class PhabricatorDaemonTaskGarbageCollector
extends PhabricatorGarbageCollector {
public function collectGarbage() {
$key = 'gcdaemon.ttl.task-archive';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return false;
}
$table = new PhabricatorWorkerArchiveTask();
$data_table = new PhabricatorWorkerTaskData();
$conn_w = $table->establishConnection('w');
$rows = queryfx_all(
$conn_w,
'SELECT id, dataID FROM %T WHERE dateCreated < %d LIMIT 100',
$table->getTableName(),
time() - $ttl);
if (!$rows) {
return false;
}
$data_ids = array_filter(ipull($rows, 'dataID'));
$task_ids = ipull($rows, 'id');
$table->openTransaction();
if ($data_ids) {
queryfx(
$conn_w,
'DELETE FROM %T WHERE id IN (%Ld)',
$data_table->getTableName(),
$data_ids);
}
queryfx(
$conn_w,
'DELETE FROM %T WHERE id IN (%Ld)',
$table->getTableName(),
$task_ids);
$table->saveTransaction();
return (count($task_ids) == 100);
}
}

View file

@ -0,0 +1,25 @@
<?php
final class DifferentialParseCacheGarbageCollector
extends PhabricatorGarbageCollector {
public function collectGarbage() {
$key = 'gcdaemon.ttl.differential-parse-cache';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return false;
}
$table = new DifferentialChangeset();
$conn_w = $table->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE dateCreated < %d LIMIT 100',
DifferentialChangeset::TABLE_CACHE,
time() - $ttl);
return ($conn_w->getAffectedRows() == 100);
}
}

View file

@ -0,0 +1,18 @@
<?php
final class PhabricatorFileTemporaryGarbageCollector
extends PhabricatorGarbageCollector {
public function collectGarbage() {
$files = id(new PhabricatorFile())->loadAllWhere(
'ttl < %d LIMIT 100',
time());
foreach ($files as $file) {
$file->delete();
}
return (count($files) == 100);
}
}

View file

@ -3,8 +3,6 @@
/**
* Collects old logs and caches to reduce the amount of data stored in the
* database.
*
* @group daemon
*/
final class PhabricatorGarbageCollectorDaemon extends PhabricatorDaemon {
@ -14,40 +12,6 @@ final class PhabricatorGarbageCollectorDaemon extends PhabricatorDaemon {
->loadObjects();
do {
$n_daemon = $this->collectDaemonLogs();
$n_parse = $this->collectParseCaches();
$n_markup = $this->collectMarkupCaches();
$n_tasks = $this->collectArchivedTasks();
$n_cache_ttl = $this->collectGeneralCacheTTL();
$n_cache = $this->collectGeneralCaches();
$n_files = $this->collectExpiredFiles();
$n_clogs = $this->collectExpiredConduitLogs();
$n_ccons = $this->collectExpiredConduitConnections();
$collected = array(
'Daemon Log' => $n_daemon,
'Differential Parse Cache' => $n_parse,
'Markup Cache' => $n_markup,
'Archived Tasks' => $n_tasks,
'General Cache TTL' => $n_cache_ttl,
'General Cache Entries' => $n_cache,
'Temporary Files' => $n_files,
'Conduit Logs' => $n_clogs,
'Conduit Connections' => $n_ccons,
);
$collected = array_filter($collected);
foreach ($collected as $thing => $count) {
$count = number_format($count);
$this->log("Garbage collected {$count} '{$thing}' objects.");
}
$total = array_sum($collected);
// TODO: This logic is unnecessarily complex for now to facilitate a
// gradual conversion to the new GC infrastructure.
$had_more_garbage = false;
foreach ($collectors as $name => $collector) {
$more_garbage = false;
do {
@ -58,210 +22,17 @@ final class PhabricatorGarbageCollectorDaemon extends PhabricatorDaemon {
}
$more_garbage = $collector->collectGarbage();
if ($more_garbage) {
$had_more_garbage = true;
}
$this->stillWorking();
} while ($more_garbage);
}
if ($had_more_garbage) {
$total += 100;
}
if ($total < 100) {
// We didn't max out any of the GCs so we're basically caught up. Ease
// off the GC loop so we don't keep doing table scans just to delete
// a handful of rows; wake up in a few hours.
$this->sleep(4 * (60 * 60));
} else {
$this->stillWorking();
}
// We made it to the end of the run cycle of every GC, so we're more or
// less caught up. Ease off the GC loop so we don't keep doing table
// scans just to delete a handful of rows; wake up in a few hours.
$this->log(pht('All caught up, waiting for more garbage.'));
$this->sleep(4 * (60 * 60));
} while (true);
}
private function collectDaemonLogs() {
$ttl = PhabricatorEnv::getEnvConfig('gcdaemon.ttl.daemon-logs');
if ($ttl <= 0) {
return 0;
}
$table = new PhabricatorDaemonLogEvent();
$conn_w = $table->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE epoch < %d LIMIT 100',
$table->getTableName(),
time() - $ttl);
return $conn_w->getAffectedRows();
}
private function collectParseCaches() {
$key = 'gcdaemon.ttl.differential-parse-cache';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return 0;
}
$table = new DifferentialChangeset();
$conn_w = $table->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE dateCreated < %d LIMIT 100',
DifferentialChangeset::TABLE_CACHE,
time() - $ttl);
return $conn_w->getAffectedRows();
}
private function collectMarkupCaches() {
$key = 'gcdaemon.ttl.markup-cache';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return 0;
}
$table = new PhabricatorMarkupCache();
$conn_w = $table->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE dateCreated < %d LIMIT 100',
$table->getTableName(),
time() - $ttl);
return $conn_w->getAffectedRows();
}
private function collectArchivedTasks() {
$key = 'gcdaemon.ttl.task-archive';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return 0;
}
$table = new PhabricatorWorkerArchiveTask();
$data_table = new PhabricatorWorkerTaskData();
$conn_w = $table->establishConnection('w');
$rows = queryfx_all(
$conn_w,
'SELECT id, dataID FROM %T WHERE dateCreated < %d LIMIT 100',
$table->getTableName(),
time() - $ttl);
if (!$rows) {
return 0;
}
$data_ids = array_filter(ipull($rows, 'dataID'));
$task_ids = ipull($rows, 'id');
$table->openTransaction();
if ($data_ids) {
queryfx(
$conn_w,
'DELETE FROM %T WHERE id IN (%Ld)',
$data_table->getTableName(),
$data_ids);
}
queryfx(
$conn_w,
'DELETE FROM %T WHERE id IN (%Ld)',
$table->getTableName(),
$task_ids);
$table->saveTransaction();
return count($task_ids);
}
private function collectGeneralCacheTTL() {
$cache = new PhabricatorKeyValueDatabaseCache();
$conn_w = $cache->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE cacheExpires < %d
ORDER BY cacheExpires ASC LIMIT 100',
$cache->getTableName(),
time());
return $conn_w->getAffectedRows();
}
private function collectGeneralCaches() {
$key = 'gcdaemon.ttl.general-cache';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return 0;
}
$cache = new PhabricatorKeyValueDatabaseCache();
$conn_w = $cache->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE cacheCreated < %d
ORDER BY cacheCreated ASC LIMIT 100',
$cache->getTableName(),
time() - $ttl);
return $conn_w->getAffectedRows();
}
private function collectExpiredFiles() {
$files = id(new PhabricatorFile())->loadAllWhere('ttl < %d LIMIT 100',
time());
foreach ($files as $file) {
$file->delete();
}
return count($files);
}
private function collectExpiredConduitLogs() {
$key = 'gcdaemon.ttl.conduit-logs';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return 0;
}
$table = new PhabricatorConduitMethodCallLog();
$conn_w = $table->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE dateCreated < %d
ORDER BY dateCreated ASC LIMIT 100',
$table->getTableName(),
time() - $ttl);
return $conn_w->getAffectedRows();
}
private function collectExpiredConduitConnections() {
$key = 'gcdaemon.ttl.conduit-logs';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return 0;
}
$table = new PhabricatorConduitConnectionLog();
$conn_w = $table->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE dateCreated < %d
ORDER BY dateCreated ASC LIMIT 100',
$table->getTableName(),
time() - $ttl);
return $conn_w->getAffectedRows();
}
}