mirror of
https://we.phorge.it/source/phorge.git
synced 2024-11-26 16:52:41 +01:00
Send graceful shutdown signals to daemons in Phabricator
Summary: Fixes T5855. Adds a `--graceful N` flag to `phd stop` and `phd restart`. `phd` will send SIGINT, wait `N` seconds, SIGTERM, wait 15 seconds, and SIGKILL. By default, `N` is 15. Test Plan: - Ran `bin/phd debug ...` and used `^C` to interrupt daemons. Saw graceful shutdown behavior, and abrupt termination on multiple `^C`. - Ran `bin/phd start`, `bin/phd stop` and `bin/phd restart` with `--graceful` set to various things, notably `0`. Saw graceful shutdowns on the CLI and in the web UI. With `0`, abrupt shutdowns. Reviewers: btrahan, hach-que Reviewed By: hach-que Subscribers: epriestley Maniphest Tasks: T5855 Differential Revision: https://secure.phabricator.com/D10228
This commit is contained in:
parent
aab0ed1c50
commit
9309723ac4
13 changed files with 117 additions and 41 deletions
|
@ -53,6 +53,10 @@ final class PhabricatorDaemonLogViewController
|
|||
$tag->setBackgroundColor(PHUITagView::COLOR_BLUE);
|
||||
$tag->setName(pht('Waiting'));
|
||||
break;
|
||||
case PhabricatorDaemonLog::STATUS_EXITING:
|
||||
$tag->setBackgroundColor(PHUITagView::COLOR_YELLOW);
|
||||
$tag->setName(pht('Exiting'));
|
||||
break;
|
||||
case PhabricatorDaemonLog::STATUS_EXITED:
|
||||
$tag->setBackgroundColor(PHUITagView::COLOR_GREY);
|
||||
$tag->setName(pht('Exited'));
|
||||
|
@ -136,6 +140,10 @@ final class PhabricatorDaemonLogViewController
|
|||
phutil_format_relative_time($unknown_time),
|
||||
phutil_format_relative_time($wait_time));
|
||||
break;
|
||||
case PhabricatorDaemonLog::STATUS_EXITING:
|
||||
$details = pht(
|
||||
'This daemon is shutting down gracefully.');
|
||||
break;
|
||||
case PhabricatorDaemonLog::STATUS_EXITED:
|
||||
$details = pht(
|
||||
'This daemon exited normally and is no longer running.');
|
||||
|
|
|
@ -8,6 +8,7 @@ final class PhabricatorDaemonEventListener extends PhabricatorEventListener {
|
|||
$this->listen(PhutilDaemonOverseer::EVENT_DID_LAUNCH);
|
||||
$this->listen(PhutilDaemonOverseer::EVENT_DID_LOG);
|
||||
$this->listen(PhutilDaemonOverseer::EVENT_DID_HEARTBEAT);
|
||||
$this->listen(PhutilDaemonOverseer::EVENT_WILL_GRACEFUL);
|
||||
$this->listen(PhutilDaemonOverseer::EVENT_WILL_EXIT);
|
||||
}
|
||||
|
||||
|
@ -22,6 +23,9 @@ final class PhabricatorDaemonEventListener extends PhabricatorEventListener {
|
|||
case PhutilDaemonOverseer::EVENT_DID_LOG:
|
||||
$this->handleLogEvent($event);
|
||||
break;
|
||||
case PhutilDaemonOverseer::EVENT_WILL_GRACEFUL:
|
||||
$this->handleGracefulEvent($event);
|
||||
break;
|
||||
case PhutilDaemonOverseer::EVENT_WILL_EXIT:
|
||||
$this->handleExitEvent($event);
|
||||
break;
|
||||
|
@ -86,6 +90,13 @@ final class PhabricatorDaemonEventListener extends PhabricatorEventListener {
|
|||
}
|
||||
}
|
||||
|
||||
private function handleGracefulEvent(PhutilEvent $event) {
|
||||
$id = $event->getValue('id');
|
||||
|
||||
$daemon = $this->getDaemon($id);
|
||||
$daemon->setStatus(PhabricatorDaemonLog::STATUS_EXITING)->save();
|
||||
}
|
||||
|
||||
private function handleExitEvent(PhutilEvent $event) {
|
||||
$id = $event->getValue('id');
|
||||
|
||||
|
|
|
@ -9,11 +9,22 @@ final class PhabricatorDaemonManagementRestartWorkflow
|
|||
->setSynopsis(
|
||||
pht(
|
||||
'Stop, then start the standard daemon loadout.'))
|
||||
->setArguments(array());
|
||||
->setArguments(
|
||||
array(
|
||||
array(
|
||||
'name' => 'graceful',
|
||||
'param' => 'seconds',
|
||||
'help' => pht(
|
||||
'Grace period for daemons to attempt a clean shutdown, in '.
|
||||
'seconds. Defaults to __15__ seconds.'),
|
||||
'default' => 15,
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
public function execute(PhutilArgumentParser $args) {
|
||||
$err = $this->executeStopCommand(array());
|
||||
$graceful = $args->getArg('graceful');
|
||||
$err = $this->executeStopCommand(array(), $graceful);
|
||||
if ($err) {
|
||||
return $err;
|
||||
}
|
||||
|
|
|
@ -12,6 +12,14 @@ final class PhabricatorDaemonManagementStopWorkflow
|
|||
'Use **phd status** to find PIDs.'))
|
||||
->setArguments(
|
||||
array(
|
||||
array(
|
||||
'name' => 'graceful',
|
||||
'param' => 'seconds',
|
||||
'help' => pht(
|
||||
'Grace period for daemons to attempt a clean shutdown, in '.
|
||||
'seconds. Defaults to __15__ seconds.'),
|
||||
'default' => 15,
|
||||
),
|
||||
array(
|
||||
'name' => 'pids',
|
||||
'wildcard' => true,
|
||||
|
@ -21,7 +29,8 @@ final class PhabricatorDaemonManagementStopWorkflow
|
|||
|
||||
public function execute(PhutilArgumentParser $args) {
|
||||
$pids = $args->getArg('pids');
|
||||
return $this->executeStopCommand($pids);
|
||||
$graceful = $args->getArg('graceful');
|
||||
return $this->executeStopCommand($pids, $graceful);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -286,7 +286,7 @@ abstract class PhabricatorDaemonManagementWorkflow
|
|||
return 0;
|
||||
}
|
||||
|
||||
protected final function executeStopCommand(array $pids) {
|
||||
protected final function executeStopCommand(array $pids, $grace_period) {
|
||||
$console = PhutilConsole::getConsole();
|
||||
|
||||
$daemons = $this->loadRunningDaemons();
|
||||
|
@ -325,39 +325,20 @@ abstract class PhabricatorDaemonManagementWorkflow
|
|||
}
|
||||
|
||||
$all_daemons = $running;
|
||||
foreach ($running as $key => $daemon) {
|
||||
$pid = $daemon->getPID();
|
||||
$name = $daemon->getName();
|
||||
|
||||
$console->writeErr(pht("Stopping daemon '%s' (%s)...", $name, $pid)."\n");
|
||||
if (!$daemon->isRunning()) {
|
||||
$console->writeErr(pht('Daemon is not running.')."\n");
|
||||
unset($running[$key]);
|
||||
$daemon->updateStatus(PhabricatorDaemonLog::STATUS_EXITED);
|
||||
} else {
|
||||
posix_kill($pid, SIGINT);
|
||||
}
|
||||
// If we're doing a graceful shutdown, try SIGINT first.
|
||||
if ($grace_period) {
|
||||
$running = $this->sendSignal($running, SIGINT, $grace_period);
|
||||
}
|
||||
|
||||
$start = time();
|
||||
do {
|
||||
foreach ($running as $key => $daemon) {
|
||||
$pid = $daemon->getPID();
|
||||
if (!$daemon->isRunning()) {
|
||||
$console->writeOut(pht('Daemon %s exited normally.', $pid)."\n");
|
||||
unset($running[$key]);
|
||||
}
|
||||
}
|
||||
if (empty($running)) {
|
||||
break;
|
||||
}
|
||||
usleep(100000);
|
||||
} while (time() < $start + 15);
|
||||
// If we still have daemons, SIGTERM them.
|
||||
if ($running) {
|
||||
$running = $this->sendSignal($running, SIGTERM, 15);
|
||||
}
|
||||
|
||||
foreach ($running as $key => $daemon) {
|
||||
$pid = $daemon->getPID();
|
||||
$console->writeErr(pht('Sending daemon %s a SIGKILL.', $pid)."\n");
|
||||
posix_kill($pid, SIGKILL);
|
||||
// If the overseer is still alive, SIGKILL it.
|
||||
if ($running) {
|
||||
$this->sendSignal($running, SIGKILL, 0);
|
||||
}
|
||||
|
||||
foreach ($all_daemons as $daemon) {
|
||||
|
@ -369,6 +350,49 @@ abstract class PhabricatorDaemonManagementWorkflow
|
|||
return 0;
|
||||
}
|
||||
|
||||
private function sendSignal(array $daemons, $signo, $wait) {
|
||||
$console = PhutilConsole::getConsole();
|
||||
|
||||
foreach ($daemons as $key => $daemon) {
|
||||
$pid = $daemon->getPID();
|
||||
$name = $daemon->getName();
|
||||
|
||||
switch ($signo) {
|
||||
case SIGINT:
|
||||
$message = pht("Interrupting daemon '%s' (%s)...", $name, $pid);
|
||||
break;
|
||||
case SIGTERM:
|
||||
$message = pht("Terminating daemon '%s' (%s)...", $name, $pid);
|
||||
break;
|
||||
case SIGKILL:
|
||||
$message = pht("Killing daemon '%s' (%s)...", $name, $pid);
|
||||
break;
|
||||
}
|
||||
|
||||
$console->writeOut("%s\n", $message);
|
||||
posix_kill($pid, $signo);
|
||||
}
|
||||
|
||||
if ($wait) {
|
||||
$start = PhabricatorTime::getNow();
|
||||
do {
|
||||
foreach ($daemons as $key => $daemon) {
|
||||
$pid = $daemon->getPID();
|
||||
if (!$daemon->isRunning()) {
|
||||
$console->writeOut(pht('Daemon %s exited.', $pid)."\n");
|
||||
unset($daemons[$key]);
|
||||
}
|
||||
}
|
||||
if (empty($daemons)) {
|
||||
break;
|
||||
}
|
||||
usleep(100000);
|
||||
} while (PhabricatorTime::getNow() < $start + $wait);
|
||||
}
|
||||
|
||||
return $daemons;
|
||||
}
|
||||
|
||||
private function freeActiveLeases() {
|
||||
$task_table = id(new PhabricatorWorkerActiveTask());
|
||||
$conn_w = $task_table->establishConnection('w');
|
||||
|
|
|
@ -67,6 +67,7 @@ final class PhabricatorDaemonLogQuery
|
|||
$status_running = PhabricatorDaemonLog::STATUS_RUNNING;
|
||||
$status_unknown = PhabricatorDaemonLog::STATUS_UNKNOWN;
|
||||
$status_wait = PhabricatorDaemonLog::STATUS_WAIT;
|
||||
$status_exiting = PhabricatorDaemonLog::STATUS_EXITING;
|
||||
$status_exited = PhabricatorDaemonLog::STATUS_EXITED;
|
||||
$status_dead = PhabricatorDaemonLog::STATUS_DEAD;
|
||||
|
||||
|
@ -77,7 +78,8 @@ final class PhabricatorDaemonLogQuery
|
|||
$seen = $daemon->getDateModified();
|
||||
|
||||
$is_running = ($status == $status_running) ||
|
||||
($status == $status_wait);
|
||||
($status == $status_wait) ||
|
||||
($status == $status_exiting);
|
||||
|
||||
// If we haven't seen the daemon recently, downgrade its status to
|
||||
// unknown.
|
||||
|
@ -160,6 +162,7 @@ final class PhabricatorDaemonLogQuery
|
|||
PhabricatorDaemonLog::STATUS_UNKNOWN,
|
||||
PhabricatorDaemonLog::STATUS_RUNNING,
|
||||
PhabricatorDaemonLog::STATUS_WAIT,
|
||||
PhabricatorDaemonLog::STATUS_EXITING,
|
||||
);
|
||||
default:
|
||||
throw new Exception("Unknown status '{$status}'!");
|
||||
|
|
|
@ -7,6 +7,7 @@ final class PhabricatorDaemonLog extends PhabricatorDaemonDAO
|
|||
const STATUS_RUNNING = 'run';
|
||||
const STATUS_DEAD = 'dead';
|
||||
const STATUS_WAIT = 'wait';
|
||||
const STATUS_EXITING = 'exiting';
|
||||
const STATUS_EXITED = 'exit';
|
||||
|
||||
protected $daemon;
|
||||
|
|
|
@ -17,8 +17,7 @@ final class PhabricatorDaemonLogListView extends AphrontView {
|
|||
throw new Exception('Call setUser() before rendering!');
|
||||
}
|
||||
|
||||
$list = id(new PHUIObjectItemListView())
|
||||
->setFlush(true);
|
||||
$list = new PHUIObjectItemListView();
|
||||
foreach ($this->daemonLogs as $log) {
|
||||
$id = $log->getID();
|
||||
$epoch = $log->getDateCreated();
|
||||
|
@ -43,6 +42,10 @@ final class PhabricatorDaemonLogListView extends AphrontView {
|
|||
'dead.'));
|
||||
$item->addIcon('fa-times grey', pht('Dead'));
|
||||
break;
|
||||
case PhabricatorDaemonLog::STATUS_EXITING:
|
||||
$item->addAttribute(pht('This daemon is exiting.'));
|
||||
$item->addIcon('fa-check', pht('Exiting'));
|
||||
break;
|
||||
case PhabricatorDaemonLog::STATUS_EXITED:
|
||||
$item->setDisabled(true);
|
||||
$item->addAttribute(pht('This daemon exited cleanly.'));
|
||||
|
|
|
@ -8,7 +8,7 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon {
|
|||
|
||||
public function run() {
|
||||
$this->setEngines(PhabricatorFactEngine::loadAllEngines());
|
||||
while (true) {
|
||||
while (!$this->shouldExit()) {
|
||||
$iterators = $this->getAllApplicationIterators();
|
||||
foreach ($iterators as $iterator_name => $iterator) {
|
||||
$this->processIteratorWithCursor($iterator_name, $iterator);
|
||||
|
|
|
@ -71,7 +71,7 @@ final class PhabricatorRepositoryPullLocalDaemon
|
|||
$futures = array();
|
||||
$queue = array();
|
||||
|
||||
while (true) {
|
||||
while (!$this->shouldExit()) {
|
||||
$pullable = $this->loadPullableRepositories($include, $exclude);
|
||||
|
||||
// If any repositories have the NEEDS_UPDATE flag set, pull them
|
||||
|
@ -422,6 +422,12 @@ final class PhabricatorRepositoryPullLocalDaemon
|
|||
new PhutilNumber($sleep_duration)));
|
||||
|
||||
$this->sleep(1);
|
||||
|
||||
if ($this->shouldExit()) {
|
||||
$this->log(pht('Awakened from sleep by graceful shutdown!'));
|
||||
return;
|
||||
}
|
||||
|
||||
if ($this->loadRepositoryUpdateMessages()) {
|
||||
$this->log(pht('Awakened from sleep by pending updates!'));
|
||||
break;
|
||||
|
|
|
@ -103,7 +103,7 @@ final class PhabricatorBot extends PhabricatorDaemon {
|
|||
foreach ($this->handlers as $handler) {
|
||||
$handler->runBackgroundTasks();
|
||||
}
|
||||
} while (true);
|
||||
} while (!$this->shouldExit());
|
||||
}
|
||||
|
||||
public function writeMessage(PhabricatorBotMessage $message) {
|
||||
|
|
|
@ -31,7 +31,7 @@ final class PhabricatorGarbageCollectorDaemon extends PhabricatorDaemon {
|
|||
// scans just to delete a handful of rows; wake up in a few hours.
|
||||
$this->log(pht('All caught up, waiting for more garbage.'));
|
||||
$this->sleep(4 * (60 * 60));
|
||||
} while (true);
|
||||
} while (!$this->shouldExit());
|
||||
|
||||
}
|
||||
|
||||
|
|
|
@ -40,7 +40,7 @@ final class PhabricatorTaskmasterDaemon extends PhabricatorDaemon {
|
|||
}
|
||||
|
||||
$this->sleep($sleep);
|
||||
} while (true);
|
||||
} while (!$this->shouldExit());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue