revi-archive/phorge-phorge
1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2025-01-05 20:31:03 +01:00
Code Issues Releases Wiki Activity

Provide bin/garbage for interacting with garbage collection

Browse source 
Summary:
Fixes T9494. This:

  - Removes all the random GC.x.y.z config.
  - Puts it all in one place that's locked and which you use `bin/garbage set-policy ...` to adjust.
  - Makes every TTL-based GC configurable.
  - Simplifies the code in the actual GCs.

Test Plan:
  - Ran `bin/garbage collect` to collect some garbage, until it stopped collecting.
  - Ran `bin/garbage set-policy ...` to shorten policy. Saw change in web UI. Ran `bin/garbage collect` again and saw it collect more garbage.
  - Set policy to indefinite and saw it not collect garabge.
  - Set policy to default and saw it reflected in web UI / `collect`.
  - Ran `bin/phd debug trigger` and saw all GCs fire with reasonable looking queries.
  - Read new docs.

{F857928}

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T9494

Differential Revision: https://secure.phabricator.com/D14219
This commit is contained in:
epriestley 2015-10-02 09:17:24 -07:00
parent bb4667cb84
commit 9c798e5cca
36 changed files with 486 additions and 188 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
bin/garbage Symbolic link
View file

@ -0,0 +1 @@
../scripts/setup/manage_garbage.php

21
scripts/setup/manage_garbage.php Executable file
View file

@ -0,0 +1,21 @@
#!/usr/bin/env php
<?php
$root = dirname(dirname(dirname(__FILE__)));
require_once $root.'/scripts/__init_script__.php';
$args = new PhutilArgumentParser($argv);
$args->setTagline(pht('manage garbage colletors'));
$args->setSynopsis(<<<EOSYNOPSIS
**garbage** __command__ [__options__]
Manage garbage collectors.
EOSYNOPSIS
);
$args->parseStandardArguments();
$workflows = id(new PhutilClassMapQuery())
->setAncestorClass('PhabricatorGarbageCollectorManagementWorkflow')
->execute();
$workflows[] = new PhutilHelpArgumentWorkflow();
$args->parseWorkflows($workflows);

8
src/__phutil_library_map__.php
View file

@ -2197,7 +2197,9 @@ phutil_register_library_map(array(
'PhabricatorFundApplication' => 'applications/fund/application/PhabricatorFundApplication.php',
'PhabricatorGDSetupCheck' => 'applications/config/check/PhabricatorGDSetupCheck.php',
'PhabricatorGarbageCollector' => 'infrastructure/daemon/garbagecollector/PhabricatorGarbageCollector.php',
'PhabricatorGarbageCollectorConfigOptions' => 'applications/config/option/PhabricatorGarbageCollectorConfigOptions.php',
'PhabricatorGarbageCollectorManagementCollectWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCollectWorkflow.php',
'PhabricatorGarbageCollectorManagementSetPolicyWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementSetPolicyWorkflow.php',
'PhabricatorGarbageCollectorManagementWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementWorkflow.php',
'PhabricatorGestureUIExample' => 'applications/uiexample/examples/PhabricatorGestureUIExample.php',
'PhabricatorGitGraphStream' => 'applications/repository/daemon/PhabricatorGitGraphStream.php',
'PhabricatorGitHubAuthProvider' => 'applications/auth/provider/PhabricatorGitHubAuthProvider.php',
@ -6197,7 +6199,9 @@ phutil_register_library_map(array(
'PhabricatorFundApplication' => 'PhabricatorApplication',
'PhabricatorGDSetupCheck' => 'PhabricatorSetupCheck',
'PhabricatorGarbageCollector' => 'Phobject',
'PhabricatorGarbageCollectorConfigOptions' => 'PhabricatorApplicationConfigOptions',
'PhabricatorGarbageCollectorManagementCollectWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow',
'PhabricatorGarbageCollectorManagementSetPolicyWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow',
'PhabricatorGarbageCollectorManagementWorkflow' => 'PhabricatorManagementWorkflow',
'PhabricatorGestureUIExample' => 'PhabricatorUIExample',
'PhabricatorGitGraphStream' => 'PhabricatorRepositoryGraphStream',
'PhabricatorGitHubAuthProvider' => 'PhabricatorOAuth2AuthProvider',

4
src/applications/auth/garbagecollector/PhabricatorAuthSessionGarbageCollector.php
View file

@ -6,14 +6,14 @@ final class PhabricatorAuthSessionGarbageCollector
const COLLECTORCONST = 'auth.sessions';
public function getCollectorName() {
return pht('Auth Sessions');
return pht('Authentication Sessions');
}
public function hasAutomaticPolicy() {
return true;
}
public function collectGarbage() {
protected function collectGarbage() {
$session_table = new PhabricatorAuthSession();
$conn_w = $session_table->establishConnection('w');

4
src/applications/auth/garbagecollector/PhabricatorAuthTemporaryTokenGarbageCollector.php
View file

@ -6,14 +6,14 @@ final class PhabricatorAuthTemporaryTokenGarbageCollector
const COLLECTORCONST = 'auth.tokens';
public function getCollectorName() {
return pht('Auth Tokens');
return pht('Authentication Tokens');
}
public function hasAutomaticPolicy() {
return true;
}
public function collectGarbage() {
protected function collectGarbage() {
$session_table = new PhabricatorAuthTemporaryToken();
$conn_w = $session_table->establishConnection('w');

10
src/applications/cache/garbagecollector/PhabricatorCacheGeneralGarbageCollector.php vendored
View file

@ -13,13 +13,7 @@ final class PhabricatorCacheGeneralGarbageCollector
return phutil_units('30 days in seconds');
}
public function collectGarbage() {
$key = 'gcdaemon.ttl.general-cache';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return false;
}
protected function collectGarbage() {
$cache = new PhabricatorKeyValueDatabaseCache();
$conn_w = $cache->establishConnection('w');
@ -28,7 +22,7 @@ final class PhabricatorCacheGeneralGarbageCollector
'DELETE FROM %T WHERE cacheCreated < %d
ORDER BY cacheCreated ASC LIMIT 100',
$cache->getTableName(),
time() - $ttl);
$this->getGarbageEpoch());
return ($conn_w->getAffectedRows() == 100);
}

10
src/applications/cache/garbagecollector/PhabricatorCacheMarkupGarbageCollector.php vendored
View file

@ -13,13 +13,7 @@ final class PhabricatorCacheMarkupGarbageCollector
return phutil_units('30 days in seconds');
}
public function collectGarbage() {
$key = 'gcdaemon.ttl.markup-cache';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return false;
}
protected function collectGarbage() {
$table = new PhabricatorMarkupCache();
$conn_w = $table->establishConnection('w');
@ -27,7 +21,7 @@ final class PhabricatorCacheMarkupGarbageCollector
$conn_w,
'DELETE FROM %T WHERE dateCreated < %d LIMIT 100',
$table->getTableName(),
time() - $ttl);
$this->getGarbageEpoch());
return ($conn_w->getAffectedRows() == 100);
}

4
src/applications/cache/garbagecollector/PhabricatorCacheTTLGarbageCollector.php vendored
View file

@ -13,7 +13,7 @@ final class PhabricatorCacheTTLGarbageCollector
return true;
}
public function collectGarbage() {
protected function collectGarbage() {
$cache = new PhabricatorKeyValueDatabaseCache();
$conn_w = $cache->establishConnection('w');
@ -22,7 +22,7 @@ final class PhabricatorCacheTTLGarbageCollector
'DELETE FROM %T WHERE cacheExpires < %d
ORDER BY cacheExpires ASC LIMIT 100',
$cache->getTableName(),
time());
PhabricatorTime::getNow());
return ($conn_w->getAffectedRows() == 100);
}

11
src/applications/conduit/garbagecollector/ConduitConnectionGarbageCollector.php
View file

@ -13,21 +13,16 @@ final class ConduitConnectionGarbageCollector
return phutil_units('180 days in seconds');
}
public function collectGarbage() {
$key = 'gcdaemon.ttl.conduit-logs';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return false;
}
protected function collectGarbage() {
$table = new PhabricatorConduitConnectionLog();
$conn_w = $table->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE dateCreated < %d
ORDER BY dateCreated ASC LIMIT 100',
$table->getTableName(),
time() - $ttl);
$this->getGarbageEpoch());
return ($conn_w->getAffectedRows() == 100);
}

11
src/applications/conduit/garbagecollector/ConduitLogGarbageCollector.php
View file

@ -13,21 +13,16 @@ final class ConduitLogGarbageCollector
return phutil_units('180 days in seconds');
}
public function collectGarbage() {
$key = 'gcdaemon.ttl.conduit-logs';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return false;
}
protected function collectGarbage() {
$table = new PhabricatorConduitMethodCallLog();
$conn_w = $table->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE dateCreated < %d
ORDER BY dateCreated ASC LIMIT 100',
$table->getTableName(),
time() - $ttl);
$this->getGarbageEpoch());
return ($conn_w->getAffectedRows() == 100);
}

3
src/applications/conduit/garbagecollector/ConduitTokenGarbageCollector.php
View file

@ -13,9 +13,10 @@ final class ConduitTokenGarbageCollector
return true;
}
public function collectGarbage() {
protected function collectGarbage() {
$table = new PhabricatorConduitToken();
$conn_w = $table->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE expires <= %d

12
src/applications/config/check/PhabricatorExtraConfigSetupCheck.php
View file

@ -176,6 +176,10 @@ final class PhabricatorExtraConfigSetupCheck extends PhabricatorSetupCheck {
'Inbound mail addresses are now configured for each application '.
'in the Applications tool.');
$gc_reason = pht(
'Garbage collectors are now configured with "%s".',
'bin/garbage set-policy');
$ancient_config += array(
'phid.external-loaders' =>
pht(
@ -280,6 +284,14 @@ final class PhabricatorExtraConfigSetupCheck extends PhabricatorSetupCheck {
'auth.login-message' => pht(
'This configuration option has been replaced with a modular '.
'handler. See T9346.'),
'gcdaemon.ttl.herald-transcripts' => $gc_reason,
'gcdaemon.ttl.daemon-logs' => $gc_reason,
'gcdaemon.ttl.differential-parse-cache' => $gc_reason,
'gcdaemon.ttl.markup-cache' => $gc_reason,
'gcdaemon.ttl.task-archive' => $gc_reason,
'gcdaemon.ttl.general-cache' => $gc_reason,
'gcdaemon.ttl.conduit-logs' => $gc_reason,
);
return $ancient_config;

22
src/applications/config/module/PhabricatorConfigCollectorsModule.php
View file

@ -17,11 +17,13 @@ final class PhabricatorConfigCollectorsModule extends PhabricatorConfigModule {
$collectors = msort($collectors, 'getCollectorConstant');
$rows = array();
$rowc = array();
foreach ($collectors as $key => $collector) {
$class = null;
if ($collector->hasAutomaticPolicy()) {
$policy_view = phutil_tag('em', array(), pht('Automatic'));
} else {
$policy = $collector->getDefaultRetentionPolicy();
$policy = $collector->getRetentionPolicy();
if ($policy === null) {
$policy_view = pht('Indefinite');
} else {
@ -30,8 +32,15 @@ final class PhabricatorConfigCollectorsModule extends PhabricatorConfigModule {
'%s Day(s)',
new PhutilNumber($days));
}
$default = $collector->getDefaultRetentionPolicy();
if ($policy !== $default) {
$class = 'highlighted';
$policy_view = phutil_tag('strong', array(), $policy_view);
}
}
$rowc[] = $class;
$rows[] = array(
$collector->getCollectorConstant(),
$collector->getCollectorName(),
@ -40,6 +49,7 @@ final class PhabricatorConfigCollectorsModule extends PhabricatorConfigModule {
}
$table = id(new AphrontTableView($rows))
->setRowClasses($rowc)
->setHeaders(
array(
pht('Constant'),
@ -53,8 +63,16 @@ final class PhabricatorConfigCollectorsModule extends PhabricatorConfigModule {
null,
));
$header = id(new PHUIHeaderView())
->setHeader(pht('Garbage Collectors'))
->setSubheader(
pht(
'Collectors with custom policies are highlighted. Use '.
'%s to change retention policies.',
phutil_tag('tt', array(), 'bin/garbage set-policy')));
return id(new PHUIObjectBoxView())
->setHeaderText(pht('Garbage Collectors'))
->setHeader($header)
->setTable($table);
}

70
src/applications/config/option/PhabricatorGarbageCollectorConfigOptions.php
View file

@ -1,70 +0,0 @@
<?php
final class PhabricatorGarbageCollectorConfigOptions
extends PhabricatorApplicationConfigOptions {
public function getName() {
return pht('Garbage Collector');
}
public function getDescription() {
return pht('Configure the GC for old logs, caches, etc.');
}
public function getFontIcon() {
return 'fa-trash-o';
}
public function getGroup() {
return 'core';
}
public function getOptions() {
$options = array(
'gcdaemon.ttl.herald-transcripts' => array(
30,
pht('Number of seconds to retain Herald transcripts for.'),
),
'gcdaemon.ttl.daemon-logs' => array(
7,
pht('Number of seconds to retain Daemon logs for.'),
),
'gcdaemon.ttl.differential-parse-cache' => array(
14,
pht('Number of seconds to retain Differential parse caches for.'),
),
'gcdaemon.ttl.markup-cache' => array(
30,
pht('Number of seconds to retain Markup cache entries for.'),
),
'gcdaemon.ttl.task-archive' => array(
14,
pht('Number of seconds to retain archived background tasks for.'),
),
'gcdaemon.ttl.general-cache' => array(
30,
pht('Number of seconds to retain general cache entries for.'),
),
'gcdaemon.ttl.conduit-logs' => array(
180,
pht('Number of seconds to retain Conduit call logs for.'),
),
);
$result = array();
foreach ($options as $key => $spec) {
list($default_days, $description) = $spec;
$result[] = $this
->newOption($key, 'int', $default_days * (24 * 60 * 60))
->setDescription($description)
->addExample((7 * 24 * 60 * 60), pht('Retain for 1 week'))
->addExample((14 * 24 * 60 * 60), pht('Retain for 2 weeks'))
->addExample((30 * 24 * 60 * 60), pht('Retain for 30 days'))
->addExample((60 * 24 * 60 * 60), pht('Retain for 60 days'))
->addExample(0, pht('Retain indefinitely'));
}
return $result;
}
}

11
src/applications/config/option/PhabricatorPHDConfigOptions.php
View file

@ -80,6 +80,17 @@ final class PhabricatorPHDConfigOptions
'and the daemons. Primarily, this is a way to suppress the '.
'"Daemons and Web Have Different Config" setup issue on a per '.
'config key basis.')),
$this->newOption('phd.garbage-collection', 'wild', array())
->setLocked(true)
->setLockedMessage(
pht(
'This option can not be edited from the web UI. Use %s to adjust '.
'garbage collector policies.',
phutil_tag('tt', array(), 'bin/garbage set-policy')))
->setSummary(pht('Retention policies for garbage collection.'))
->setDescription(
pht(
'Customizes retention policies for garbage collectors.')),
);
}

9
src/applications/daemon/garbagecollector/PhabricatorDaemonLogEventGarbageCollector.php
View file

@ -13,12 +13,7 @@ final class PhabricatorDaemonLogEventGarbageCollector
return phutil_units('7 days in seconds');
}
public function collectGarbage() {
$ttl = PhabricatorEnv::getEnvConfig('gcdaemon.ttl.daemon-logs');
if ($ttl <= 0) {
return false;
}
protected function collectGarbage() {
$table = new PhabricatorDaemonLogEvent();
$conn_w = $table->establishConnection('w');
@ -26,7 +21,7 @@ final class PhabricatorDaemonLogEventGarbageCollector
$conn_w,
'DELETE FROM %T WHERE epoch < %d LIMIT 100',
$table->getTableName(),
time() - $ttl);
$this->getGarbageEpoch());
return ($conn_w->getAffectedRows() == 100);
}

9
src/applications/daemon/garbagecollector/PhabricatorDaemonLogGarbageCollector.php
View file

@ -13,12 +13,7 @@ final class PhabricatorDaemonLogGarbageCollector
return phutil_units('7 days in seconds');
}
public function collectGarbage() {
$ttl = PhabricatorEnv::getEnvConfig('gcdaemon.ttl.daemon-logs');
if ($ttl <= 0) {
return false;
}
protected function collectGarbage() {
$table = new PhabricatorDaemonLog();
$conn_w = $table->establishConnection('w');
@ -26,7 +21,7 @@ final class PhabricatorDaemonLogGarbageCollector
$conn_w,
'DELETE FROM %T WHERE dateCreated < %d AND status != %s LIMIT 100',
$table->getTableName(),
time() - $ttl,
$this->getGarbageEpoch(),
PhabricatorDaemonLog::STATUS_RUNNING);
return ($conn_w->getAffectedRows() == 100);

12
src/applications/daemon/garbagecollector/PhabricatorDaemonTaskGarbageCollector.php
View file

@ -13,21 +13,15 @@ final class PhabricatorDaemonTaskGarbageCollector
return phutil_units('14 days in seconds');
}
public function collectGarbage() {
$key = 'gcdaemon.ttl.task-archive';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return false;
}
protected function collectGarbage() {
$table = new PhabricatorWorkerArchiveTask();
$data_table = new PhabricatorWorkerTaskData();
$conn_w = $table->establishConnection('w');
$tasks = id(new PhabricatorWorkerArchiveTaskQuery())
->withDateCreatedBefore(time() - $ttl)
->withDateCreatedBefore($this->getGarbageEpoch())
->setLimit(100)
->execute();
if (!$tasks) {
return false;
}

10
src/applications/differential/garbagecollector/DifferentialParseCacheGarbageCollector.php
View file

@ -13,13 +13,7 @@ final class DifferentialParseCacheGarbageCollector
return phutil_units('14 days in seconds');
}
public function collectGarbage() {
$key = 'gcdaemon.ttl.differential-parse-cache';
$ttl = PhabricatorEnv::getEnvConfig($key);
if ($ttl <= 0) {
return false;
}
protected function collectGarbage() {
$table = new DifferentialChangeset();
$conn_w = $table->establishConnection('w');
@ -27,7 +21,7 @@ final class DifferentialParseCacheGarbageCollector
$conn_w,
'DELETE FROM %T WHERE dateCreated < %d LIMIT 100',
DifferentialChangeset::TABLE_CACHE,
time() - $ttl);
$this->getGarbageEpoch());
return ($conn_w->getAffectedRows() == 100);
}

7
src/applications/drydock/garbagecollector/DrydockLogGarbageCollector.php
View file

@ -13,18 +13,15 @@ final class DrydockLogGarbageCollector
return phutil_units('30 days in seconds');
}
public function collectGarbage() {
protected function collectGarbage() {
$log_table = new DrydockLog();
$conn_w = $log_table->establishConnection('w');
$now = PhabricatorTime::getNow();
$ttl = phutil_units('30 days in seconds');
queryfx(
$conn_w,
'DELETE FROM %T WHERE epoch <= %d LIMIT 100',
$log_table->getTableName(),
$now - $ttl);
$this->getGarbageEpoch());
return ($conn_w->getAffectedRows() == 100);
}

4
src/applications/files/garbagecollector/PhabricatorFileTemporaryGarbageCollector.php
View file

@ -13,10 +13,10 @@ final class PhabricatorFileTemporaryGarbageCollector
return true;
}
public function collectGarbage() {
protected function collectGarbage() {
$files = id(new PhabricatorFile())->loadAllWhere(
'ttl < %d LIMIT 100',
time());
PhabricatorTime::getNow());
foreach ($files as $file) {
$file->delete();

9
src/applications/herald/garbagecollector/HeraldTranscriptGarbageCollector.php
View file

@ -13,12 +13,7 @@ final class HeraldTranscriptGarbageCollector
return phutil_units('30 days in seconds');
}
public function collectGarbage() {
$ttl = PhabricatorEnv::getEnvConfig('gcdaemon.ttl.herald-transcripts');
if ($ttl <= 0) {
return false;
}
protected function collectGarbage() {
$table = new HeraldTranscript();
$conn_w = $table->establishConnection('w');
@ -33,7 +28,7 @@ final class HeraldTranscriptGarbageCollector
WHERE garbageCollected = 0 AND time < %d
LIMIT 100',
$table->getTableName(),
time() - $ttl);
$this->getGarbageEpoch());
return ($conn_w->getAffectedRows() == 100);
}

6
src/applications/metamta/garbagecollector/MetaMTAMailReceivedGarbageCollector.php
View file

@ -13,9 +13,7 @@ final class MetaMTAMailReceivedGarbageCollector
return phutil_units('90 days in seconds');
}
public function collectGarbage() {
$ttl = phutil_units('90 days in seconds');
protected function collectGarbage() {
$table = new PhabricatorMetaMTAReceivedMail();
$conn_w = $table->establishConnection('w');
@ -23,7 +21,7 @@ final class MetaMTAMailReceivedGarbageCollector
$conn_w,
'DELETE FROM %T WHERE dateCreated < %d LIMIT 100',
$table->getTableName(),
time() - $ttl);
$this->getGarbageEpoch());
return ($conn_w->getAffectedRows() == 100);
}

6
src/applications/metamta/garbagecollector/MetaMTAMailSentGarbageCollector.php
View file

@ -13,12 +13,10 @@ final class MetaMTAMailSentGarbageCollector
return phutil_units('90 days in seconds');
}
public function collectGarbage() {
$ttl = phutil_units('90 days in seconds');
protected function collectGarbage() {
$mails = id(new PhabricatorMetaMTAMail())->loadAllWhere(
'dateCreated < %d LIMIT 100',
PhabricatorTime::getNow() - $ttl);
$this->getGarbageEpoch());
foreach ($mails as $mail) {
$mail->delete();

6
src/applications/multimeter/garbagecollector/MultimeterEventGarbageCollector.php
View file

@ -13,9 +13,7 @@ final class MultimeterEventGarbageCollector
return phutil_units('90 days in seconds');
}
public function collectGarbage() {
$ttl = phutil_units('90 days in seconds');
protected function collectGarbage() {
$table = new MultimeterEvent();
$conn_w = $table->establishConnection('w');
@ -23,7 +21,7 @@ final class MultimeterEventGarbageCollector
$conn_w,
'DELETE FROM %T WHERE epoch < %d LIMIT 100',
$table->getTableName(),
PhabricatorTime::getNow() - $ttl);
$this->getGarbageEpoch());
return ($conn_w->getAffectedRows() == 100);
}

6
src/applications/notification/garbagecollector/FeedStoryNotificationGarbageCollector.php
View file

@ -13,9 +13,7 @@ final class FeedStoryNotificationGarbageCollector
return phutil_units('90 days in seconds');
}
public function collectGarbage() {
$ttl = 90 * 24 * 60 * 60;
protected function collectGarbage() {
$table = new PhabricatorFeedStoryNotification();
$conn_w = $table->establishConnection('w');
@ -24,7 +22,7 @@ final class FeedStoryNotificationGarbageCollector
'DELETE FROM %T WHERE chronologicalKey < (%d << 32)
ORDER BY chronologicalKey ASC LIMIT 100',
$table->getTableName(),
time() - $ttl);
$this->getGarbageEpoch());
return ($conn_w->getAffectedRows() == 100);
}

6
src/applications/people/garbagecollector/PeopleUserLogGarbageCollector.php
View file

@ -13,9 +13,7 @@ final class PeopleUserLogGarbageCollector
return phutil_units('180 days in seconds');
}
public function collectGarbage() {
$ttl = phutil_units('180 days in seconds');
protected function collectGarbage() {
$table = new PhabricatorUserLog();
$conn_w = $table->establishConnection('w');
@ -23,7 +21,7 @@ final class PeopleUserLogGarbageCollector
$conn_w,
'DELETE FROM %T WHERE dateCreated < %d LIMIT 100',
$table->getTableName(),
time() - $ttl);
$this->getGarbageEpoch());
return ($conn_w->getAffectedRows() == 100);
}

6
src/applications/system/garbagecollector/PhabricatorSystemActionGarbageCollector.php
View file

@ -13,9 +13,7 @@ final class PhabricatorSystemActionGarbageCollector
return phutil_units('3 days in seconds');
}
public function collectGarbage() {
$ttl = phutil_units('3 days in seconds');
protected function collectGarbage() {
$table = new PhabricatorSystemActionLog();
$conn_w = $table->establishConnection('w');
@ -23,7 +21,7 @@ final class PhabricatorSystemActionGarbageCollector
$conn_w,
'DELETE FROM %T WHERE epoch < %d LIMIT 100',
$table->getTableName(),
time() - $ttl);
$this->getGarbageEpoch());
return ($conn_w->getAffectedRows() == 100);
}

6
src/applications/system/garbagecollector/PhabricatorSystemDestructionGarbageCollector.php
View file

@ -13,9 +13,7 @@ final class PhabricatorSystemDestructionGarbageCollector
return phutil_units('90 days in seconds');
}
public function collectGarbage() {
$ttl = phutil_units('90 days in seconds');
protected function collectGarbage() {
$table = new PhabricatorSystemDestructionLog();
$conn_w = $table->establishConnection('w');
@ -23,7 +21,7 @@ final class PhabricatorSystemDestructionGarbageCollector
$conn_w,
'DELETE FROM %T WHERE epoch < %d LIMIT 100',
$table->getTableName(),
time() - $ttl);
$this->getGarbageEpoch());
return ($conn_w->getAffectedRows() == 100);
}

68
src/docs/user/configuration/managing_garbage.diviner Normal file
View file

@ -0,0 +1,68 @@
@title Managing Garbage Collection
@group config
Understanding and configuring garbage collection.
Overview
========
Phabricator generates various logs and caches during normal operation. Some of
these logs and caches are usually of very little use after some time has
passed, so they are deleted automatically (often after a month or two) in a
process called "garbage collection".
Garbage collection is performed automatically by the daemons. You can review
all of the installed garbage collectors by browsing to {nav Config > Garbage
Collectors}.
Configuring Retention Policies
==============================
You can reconfigure the data retention policies for most collectors.
The default retention polcies should be suitable for most installs. However,
you might want to **decrease** retention to reduce the amount of disk space
used by some high-volume log that you don't find particularly interesting, or
to adhere to an organizational data retention policy.
Alternatively, you might want to **increase** retention if you want to retain
some logs for a longer period of time, perhaps for auditing or analytic
purposes.
You can review the current retention policies in
{nav Config > Garbage Collectors}. To change a policy, use
`bin/garbage set-policy` to select a new policy:
```
phabricator/ $ ./bin/garbage set-policy --collector cache.markup --days 7
```
You can use `--days` to select how long data is retained for. You can also use
`--indefinite` to set an indefinite retention policy. This will stop the
garbage collector from cleaning up any data. Finally, you can use `--default`
to restore the default policy.
Your changes should be reflected in the web UI immediately, and will take
effect in the actual collector **the next time the daemons are restarted**.
Troubleshooting
===============
You can manually run a collector with `bin/garbage collect`.
```
phabricator/ $ ./bin/garbage collect --collector cache.general
```
By using the `--trace` flag, you can inspect the operation of the collector
in detail.
Next Steps
==========
Continue by:
- exploring other daemon topics with @{article:Managing Daemons with phd}.

72
src/infrastructure/daemon/garbagecollector/PhabricatorGarbageCollector.php
View file

@ -46,6 +46,28 @@ abstract class PhabricatorGarbageCollector extends Phobject {
}
/**
* Get the effective retention policy.
*
* @return int|null Lifetime, or `null` for indefinite retention.
* @task info
*/
public function getRetentionPolicy() {
if ($this->hasAutomaticPolicy()) {
throw new Exception(
pht(
'Can not get retention policy of collector with automatic '.
'policy.'));
}
$config = PhabricatorEnv::getEnvConfig('phd.garbage-collection');
$const = $this->getCollectorConstant();
return idx($config, $const, $this->getDefaultRetentionPolicy());
}
/**
* Get a unique string constant identifying this collector.
*
@ -60,13 +82,61 @@ abstract class PhabricatorGarbageCollector extends Phobject {
/* -( Collecting Garbage )------------------------------------------------- */
/**
* Run the collector.
*
* @return bool True if there is more garbage to collect.
* @task collect
*/
final public function runCollector() {
// Don't do anything if this collector is configured with an indefinite
// retention policy.
if (!$this->hasAutomaticPolicy()) {
$policy = $this->getRetentionPolicy();
if (!$policy) {
return false;
}
}
return $this->collectGarbage();
}
/**
* Collect garbage from whatever source this GC handles.
*
* @return bool True if there is more garbage to collect.
* @task collect
*/
abstract public function collectGarbage();
abstract protected function collectGarbage();
/**
* Get the most recent epoch timestamp that is considered garbage.
*
* Records older than this should be collected.
*
* @return int Most recent garbage timestamp.
* @task collect
*/
final protected function getGarbageEpoch() {
if ($this->hasAutomaticPolicy()) {
throw new Exception(
pht(
'Can not get garbage epoch for a collector with an automatic '.
'collection policy.'));
}
$ttl = $this->getRetentionPolicy();
if (!$ttl) {
throw new Exception(
pht(
'Can not get garbage epoch for a collector with an indefinite '.
'retention policy.'));
}
return (PhabricatorTime::getNow() - $ttl);
}
/**

50
src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCollectWorkflow.php Normal file
View file

@ -0,0 +1,50 @@
<?php
final class PhabricatorGarbageCollectorManagementCollectWorkflow
extends PhabricatorGarbageCollectorManagementWorkflow {
protected function didConstruct() {
$this
->setName('collect')
->setExamples('**collect** --collector __collector__')
->setSynopsis(
pht('Run a garbage collector in the foreground.'))
->setArguments(
array(
array(
'name' => 'collector',
'param' => 'const',
'help' => pht(
'Constant identifying the garbage collector to run.'),
),
));
}
public function execute(PhutilArgumentParser $args) {
$collector = $this->getCollector($args->getArg('collector'));
echo tsprintf(
"%s\n",
pht('Collecting "%s" garbage...', $collector->getCollectorName()));
$any = false;
while (true) {
$more = $collector->runCollector();
if ($more) {
$any = true;
} else {
break;
}
}
if ($any) {
$message = pht('Finished collecting all the garbage.');
} else {
$message = pht('Could not find any garbage to collect.');
}
echo tsprintf("\n%s\n", $message);
return 0;
}
}

141
src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementSetPolicyWorkflow.php Normal file
View file

@ -0,0 +1,141 @@
<?php
final class PhabricatorGarbageCollectorManagementSetPolicyWorkflow
extends PhabricatorGarbageCollectorManagementWorkflow {
protected function didConstruct() {
$this
->setName('set-policy')
->setExamples(
"**set-policy** --collector __collector__ --days 30\n".
"**set-policy** --collector __collector__ --indefinite\n".
"**set-policy** --collector __collector__ --default")
->setSynopsis(
pht(
'Change retention policies for a garbage collector.'))
->setArguments(
array(
array(
'name' => 'collector',
'param' => 'const',
'help' => pht(
'Constant identifying the garbage collector.'),
),
array(
'name' => 'indefinite',
'help' => pht(
'Set an indefinite retention policy.'),
),
array(
'name' => 'default',
'help' => pht(
'Use the default retention policy.'),
),
array(
'name' => 'days',
'param' => 'count',
'help' => pht(
'Retain data for the specified number of days.'),
),
));
}
public function execute(PhutilArgumentParser $args) {
$config_key = 'phd.garbage-collection';
$collector = $this->getCollector($args->getArg('collector'));
$days = $args->getArg('days');
$indefinite = $args->getArg('indefinite');
$default = $args->getArg('default');
$count = 0;
if ($days !== null) {
$count++;
}
if ($indefinite) {
$count++;
}
if ($default) {
$count++;
}
if (!$count) {
throw new PhutilArgumentUsageException(
pht(
'Choose a policy with "%s", "%s" or "%s".',
'--days',
'--indefinite',
'--default'));
}
if ($count > 1) {
throw new PhutilArgumentUsageException(
pht(
'Options "%s", "%s" and "%s" represent mutually exclusive ways '.
'to choose a policy. Specify only one.',
'--days',
'--indefinite',
'--default'));
}
if ($days !== null) {
$days = (int)$days;
if ($days < 1) {
throw new PhutilArgumentUsageException(
pht(
'Specify a positive number of days to retain data for.'));
}
}
$collector_const = $collector->getCollectorConstant();
$value = PhabricatorEnv::getEnvConfig($config_key);
if ($days !== null) {
echo tsprintf(
"%s\n",
pht(
'Setting retention policy for "%s" to %s day(s).',
$collector->getCollectorName(),
new PhutilNumber($days)));
$value[$collector_const] = phutil_units($days.' days in seconds');
} else if ($indefinite) {
echo tsprintf(
"%s\n",
pht(
'Setting "%s" to be retained indefinitely.',
$collector->getCollectorName()));
$value[$collector_const] = null;
} else {
echo tsprintf(
"%s\n",
pht(
'Restoring "%s" to the default retention policy.',
$collector->getCollectorName()));
unset($value[$collector_const]);
}
id(new PhabricatorConfigLocalSource())
->setKeys(
array(
$config_key => $value,
));
echo tsprintf(
"%s\n",
pht(
'Wrote new policy to local configuration.'));
echo tsprintf(
"%s\n",
pht(
'This change will take effect the next time the daemons are '.
'restarted.'));
return 0;
}
}

32
src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementWorkflow.php Normal file
View file

@ -0,0 +1,32 @@
<?php
abstract class PhabricatorGarbageCollectorManagementWorkflow
extends PhabricatorManagementWorkflow {
protected function getCollector($const) {
$collectors = PhabricatorGarbageCollector::getAllCollectors();
$collector_list = array_keys($collectors);
sort($collector_list);
$collector_list = implode(', ', $collector_list);
if (!$const) {
throw new PhutilArgumentUsageException(
pht(
'Specify a collector with "%s". Valid collectors are: %s.',
'--collector',
$collector_list));
}
if (empty($collectors[$const])) {
throw new PhutilArgumentUsageException(
pht(
'No such collector "%s". Choose a valid collector: %s.',
$const,
$collector_list));
}
return $collectors[$const];
}
}

2
src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php
View file

@ -356,7 +356,7 @@ final class PhabricatorTriggerDaemon
// If we're in a collection cycle, continue collection.
if ($this->garbageCollectors) {
foreach ($this->garbageCollectors as $key => $collector) {
$more_garbage = $collector->collectGarbage();
$more_garbage = $collector->runCollector();
if (!$more_garbage) {
unset($this->garbageCollectors[$key]);
}

5
src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php
View file

@ -1393,6 +1393,11 @@ final class PhabricatorUSEnglishTranslation
'%s Days',
),
'Setting retention policy for "%s" to %s day(s).' => array(
'Setting retention policy for "%s" to one day.',
'Setting retention policy for "%s" to %s days.',
),
);
}