mirror of
https://we.phorge.it/source/phorge.git
synced 2024-11-30 02:32:42 +01:00
Send graceful shutdown signals to daemons in Phabricator
Summary: Fixes T5855. Adds a `--graceful N` flag to `phd stop` and `phd restart`. `phd` will send SIGINT, wait `N` seconds, SIGTERM, wait 15 seconds, and SIGKILL. By default, `N` is 15. Test Plan: - Ran `bin/phd debug ...` and used `^C` to interrupt daemons. Saw graceful shutdown behavior, and abrupt termination on multiple `^C`. - Ran `bin/phd start`, `bin/phd stop` and `bin/phd restart` with `--graceful` set to various things, notably `0`. Saw graceful shutdowns on the CLI and in the web UI. With `0`, abrupt shutdowns. Reviewers: btrahan, hach-que Reviewed By: hach-que Subscribers: epriestley Maniphest Tasks: T5855 Differential Revision: https://secure.phabricator.com/D10228
This commit is contained in:
parent
aab0ed1c50
commit
9309723ac4
13 changed files with 117 additions and 41 deletions
|
@ -53,6 +53,10 @@ final class PhabricatorDaemonLogViewController
|
||||||
$tag->setBackgroundColor(PHUITagView::COLOR_BLUE);
|
$tag->setBackgroundColor(PHUITagView::COLOR_BLUE);
|
||||||
$tag->setName(pht('Waiting'));
|
$tag->setName(pht('Waiting'));
|
||||||
break;
|
break;
|
||||||
|
case PhabricatorDaemonLog::STATUS_EXITING:
|
||||||
|
$tag->setBackgroundColor(PHUITagView::COLOR_YELLOW);
|
||||||
|
$tag->setName(pht('Exiting'));
|
||||||
|
break;
|
||||||
case PhabricatorDaemonLog::STATUS_EXITED:
|
case PhabricatorDaemonLog::STATUS_EXITED:
|
||||||
$tag->setBackgroundColor(PHUITagView::COLOR_GREY);
|
$tag->setBackgroundColor(PHUITagView::COLOR_GREY);
|
||||||
$tag->setName(pht('Exited'));
|
$tag->setName(pht('Exited'));
|
||||||
|
@ -136,6 +140,10 @@ final class PhabricatorDaemonLogViewController
|
||||||
phutil_format_relative_time($unknown_time),
|
phutil_format_relative_time($unknown_time),
|
||||||
phutil_format_relative_time($wait_time));
|
phutil_format_relative_time($wait_time));
|
||||||
break;
|
break;
|
||||||
|
case PhabricatorDaemonLog::STATUS_EXITING:
|
||||||
|
$details = pht(
|
||||||
|
'This daemon is shutting down gracefully.');
|
||||||
|
break;
|
||||||
case PhabricatorDaemonLog::STATUS_EXITED:
|
case PhabricatorDaemonLog::STATUS_EXITED:
|
||||||
$details = pht(
|
$details = pht(
|
||||||
'This daemon exited normally and is no longer running.');
|
'This daemon exited normally and is no longer running.');
|
||||||
|
|
|
@ -8,6 +8,7 @@ final class PhabricatorDaemonEventListener extends PhabricatorEventListener {
|
||||||
$this->listen(PhutilDaemonOverseer::EVENT_DID_LAUNCH);
|
$this->listen(PhutilDaemonOverseer::EVENT_DID_LAUNCH);
|
||||||
$this->listen(PhutilDaemonOverseer::EVENT_DID_LOG);
|
$this->listen(PhutilDaemonOverseer::EVENT_DID_LOG);
|
||||||
$this->listen(PhutilDaemonOverseer::EVENT_DID_HEARTBEAT);
|
$this->listen(PhutilDaemonOverseer::EVENT_DID_HEARTBEAT);
|
||||||
|
$this->listen(PhutilDaemonOverseer::EVENT_WILL_GRACEFUL);
|
||||||
$this->listen(PhutilDaemonOverseer::EVENT_WILL_EXIT);
|
$this->listen(PhutilDaemonOverseer::EVENT_WILL_EXIT);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -22,6 +23,9 @@ final class PhabricatorDaemonEventListener extends PhabricatorEventListener {
|
||||||
case PhutilDaemonOverseer::EVENT_DID_LOG:
|
case PhutilDaemonOverseer::EVENT_DID_LOG:
|
||||||
$this->handleLogEvent($event);
|
$this->handleLogEvent($event);
|
||||||
break;
|
break;
|
||||||
|
case PhutilDaemonOverseer::EVENT_WILL_GRACEFUL:
|
||||||
|
$this->handleGracefulEvent($event);
|
||||||
|
break;
|
||||||
case PhutilDaemonOverseer::EVENT_WILL_EXIT:
|
case PhutilDaemonOverseer::EVENT_WILL_EXIT:
|
||||||
$this->handleExitEvent($event);
|
$this->handleExitEvent($event);
|
||||||
break;
|
break;
|
||||||
|
@ -86,6 +90,13 @@ final class PhabricatorDaemonEventListener extends PhabricatorEventListener {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private function handleGracefulEvent(PhutilEvent $event) {
|
||||||
|
$id = $event->getValue('id');
|
||||||
|
|
||||||
|
$daemon = $this->getDaemon($id);
|
||||||
|
$daemon->setStatus(PhabricatorDaemonLog::STATUS_EXITING)->save();
|
||||||
|
}
|
||||||
|
|
||||||
private function handleExitEvent(PhutilEvent $event) {
|
private function handleExitEvent(PhutilEvent $event) {
|
||||||
$id = $event->getValue('id');
|
$id = $event->getValue('id');
|
||||||
|
|
||||||
|
|
|
@ -9,11 +9,22 @@ final class PhabricatorDaemonManagementRestartWorkflow
|
||||||
->setSynopsis(
|
->setSynopsis(
|
||||||
pht(
|
pht(
|
||||||
'Stop, then start the standard daemon loadout.'))
|
'Stop, then start the standard daemon loadout.'))
|
||||||
->setArguments(array());
|
->setArguments(
|
||||||
|
array(
|
||||||
|
array(
|
||||||
|
'name' => 'graceful',
|
||||||
|
'param' => 'seconds',
|
||||||
|
'help' => pht(
|
||||||
|
'Grace period for daemons to attempt a clean shutdown, in '.
|
||||||
|
'seconds. Defaults to __15__ seconds.'),
|
||||||
|
'default' => 15,
|
||||||
|
),
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
public function execute(PhutilArgumentParser $args) {
|
public function execute(PhutilArgumentParser $args) {
|
||||||
$err = $this->executeStopCommand(array());
|
$graceful = $args->getArg('graceful');
|
||||||
|
$err = $this->executeStopCommand(array(), $graceful);
|
||||||
if ($err) {
|
if ($err) {
|
||||||
return $err;
|
return $err;
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,6 +12,14 @@ final class PhabricatorDaemonManagementStopWorkflow
|
||||||
'Use **phd status** to find PIDs.'))
|
'Use **phd status** to find PIDs.'))
|
||||||
->setArguments(
|
->setArguments(
|
||||||
array(
|
array(
|
||||||
|
array(
|
||||||
|
'name' => 'graceful',
|
||||||
|
'param' => 'seconds',
|
||||||
|
'help' => pht(
|
||||||
|
'Grace period for daemons to attempt a clean shutdown, in '.
|
||||||
|
'seconds. Defaults to __15__ seconds.'),
|
||||||
|
'default' => 15,
|
||||||
|
),
|
||||||
array(
|
array(
|
||||||
'name' => 'pids',
|
'name' => 'pids',
|
||||||
'wildcard' => true,
|
'wildcard' => true,
|
||||||
|
@ -21,7 +29,8 @@ final class PhabricatorDaemonManagementStopWorkflow
|
||||||
|
|
||||||
public function execute(PhutilArgumentParser $args) {
|
public function execute(PhutilArgumentParser $args) {
|
||||||
$pids = $args->getArg('pids');
|
$pids = $args->getArg('pids');
|
||||||
return $this->executeStopCommand($pids);
|
$graceful = $args->getArg('graceful');
|
||||||
|
return $this->executeStopCommand($pids, $graceful);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -286,7 +286,7 @@ abstract class PhabricatorDaemonManagementWorkflow
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected final function executeStopCommand(array $pids) {
|
protected final function executeStopCommand(array $pids, $grace_period) {
|
||||||
$console = PhutilConsole::getConsole();
|
$console = PhutilConsole::getConsole();
|
||||||
|
|
||||||
$daemons = $this->loadRunningDaemons();
|
$daemons = $this->loadRunningDaemons();
|
||||||
|
@ -325,39 +325,20 @@ abstract class PhabricatorDaemonManagementWorkflow
|
||||||
}
|
}
|
||||||
|
|
||||||
$all_daemons = $running;
|
$all_daemons = $running;
|
||||||
foreach ($running as $key => $daemon) {
|
|
||||||
$pid = $daemon->getPID();
|
|
||||||
$name = $daemon->getName();
|
|
||||||
|
|
||||||
$console->writeErr(pht("Stopping daemon '%s' (%s)...", $name, $pid)."\n");
|
// If we're doing a graceful shutdown, try SIGINT first.
|
||||||
if (!$daemon->isRunning()) {
|
if ($grace_period) {
|
||||||
$console->writeErr(pht('Daemon is not running.')."\n");
|
$running = $this->sendSignal($running, SIGINT, $grace_period);
|
||||||
unset($running[$key]);
|
|
||||||
$daemon->updateStatus(PhabricatorDaemonLog::STATUS_EXITED);
|
|
||||||
} else {
|
|
||||||
posix_kill($pid, SIGINT);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
$start = time();
|
// If we still have daemons, SIGTERM them.
|
||||||
do {
|
if ($running) {
|
||||||
foreach ($running as $key => $daemon) {
|
$running = $this->sendSignal($running, SIGTERM, 15);
|
||||||
$pid = $daemon->getPID();
|
}
|
||||||
if (!$daemon->isRunning()) {
|
|
||||||
$console->writeOut(pht('Daemon %s exited normally.', $pid)."\n");
|
|
||||||
unset($running[$key]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (empty($running)) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
usleep(100000);
|
|
||||||
} while (time() < $start + 15);
|
|
||||||
|
|
||||||
foreach ($running as $key => $daemon) {
|
// If the overseer is still alive, SIGKILL it.
|
||||||
$pid = $daemon->getPID();
|
if ($running) {
|
||||||
$console->writeErr(pht('Sending daemon %s a SIGKILL.', $pid)."\n");
|
$this->sendSignal($running, SIGKILL, 0);
|
||||||
posix_kill($pid, SIGKILL);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach ($all_daemons as $daemon) {
|
foreach ($all_daemons as $daemon) {
|
||||||
|
@ -369,6 +350,49 @@ abstract class PhabricatorDaemonManagementWorkflow
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private function sendSignal(array $daemons, $signo, $wait) {
|
||||||
|
$console = PhutilConsole::getConsole();
|
||||||
|
|
||||||
|
foreach ($daemons as $key => $daemon) {
|
||||||
|
$pid = $daemon->getPID();
|
||||||
|
$name = $daemon->getName();
|
||||||
|
|
||||||
|
switch ($signo) {
|
||||||
|
case SIGINT:
|
||||||
|
$message = pht("Interrupting daemon '%s' (%s)...", $name, $pid);
|
||||||
|
break;
|
||||||
|
case SIGTERM:
|
||||||
|
$message = pht("Terminating daemon '%s' (%s)...", $name, $pid);
|
||||||
|
break;
|
||||||
|
case SIGKILL:
|
||||||
|
$message = pht("Killing daemon '%s' (%s)...", $name, $pid);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
$console->writeOut("%s\n", $message);
|
||||||
|
posix_kill($pid, $signo);
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($wait) {
|
||||||
|
$start = PhabricatorTime::getNow();
|
||||||
|
do {
|
||||||
|
foreach ($daemons as $key => $daemon) {
|
||||||
|
$pid = $daemon->getPID();
|
||||||
|
if (!$daemon->isRunning()) {
|
||||||
|
$console->writeOut(pht('Daemon %s exited.', $pid)."\n");
|
||||||
|
unset($daemons[$key]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (empty($daemons)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
usleep(100000);
|
||||||
|
} while (PhabricatorTime::getNow() < $start + $wait);
|
||||||
|
}
|
||||||
|
|
||||||
|
return $daemons;
|
||||||
|
}
|
||||||
|
|
||||||
private function freeActiveLeases() {
|
private function freeActiveLeases() {
|
||||||
$task_table = id(new PhabricatorWorkerActiveTask());
|
$task_table = id(new PhabricatorWorkerActiveTask());
|
||||||
$conn_w = $task_table->establishConnection('w');
|
$conn_w = $task_table->establishConnection('w');
|
||||||
|
|
|
@ -67,6 +67,7 @@ final class PhabricatorDaemonLogQuery
|
||||||
$status_running = PhabricatorDaemonLog::STATUS_RUNNING;
|
$status_running = PhabricatorDaemonLog::STATUS_RUNNING;
|
||||||
$status_unknown = PhabricatorDaemonLog::STATUS_UNKNOWN;
|
$status_unknown = PhabricatorDaemonLog::STATUS_UNKNOWN;
|
||||||
$status_wait = PhabricatorDaemonLog::STATUS_WAIT;
|
$status_wait = PhabricatorDaemonLog::STATUS_WAIT;
|
||||||
|
$status_exiting = PhabricatorDaemonLog::STATUS_EXITING;
|
||||||
$status_exited = PhabricatorDaemonLog::STATUS_EXITED;
|
$status_exited = PhabricatorDaemonLog::STATUS_EXITED;
|
||||||
$status_dead = PhabricatorDaemonLog::STATUS_DEAD;
|
$status_dead = PhabricatorDaemonLog::STATUS_DEAD;
|
||||||
|
|
||||||
|
@ -77,7 +78,8 @@ final class PhabricatorDaemonLogQuery
|
||||||
$seen = $daemon->getDateModified();
|
$seen = $daemon->getDateModified();
|
||||||
|
|
||||||
$is_running = ($status == $status_running) ||
|
$is_running = ($status == $status_running) ||
|
||||||
($status == $status_wait);
|
($status == $status_wait) ||
|
||||||
|
($status == $status_exiting);
|
||||||
|
|
||||||
// If we haven't seen the daemon recently, downgrade its status to
|
// If we haven't seen the daemon recently, downgrade its status to
|
||||||
// unknown.
|
// unknown.
|
||||||
|
@ -160,6 +162,7 @@ final class PhabricatorDaemonLogQuery
|
||||||
PhabricatorDaemonLog::STATUS_UNKNOWN,
|
PhabricatorDaemonLog::STATUS_UNKNOWN,
|
||||||
PhabricatorDaemonLog::STATUS_RUNNING,
|
PhabricatorDaemonLog::STATUS_RUNNING,
|
||||||
PhabricatorDaemonLog::STATUS_WAIT,
|
PhabricatorDaemonLog::STATUS_WAIT,
|
||||||
|
PhabricatorDaemonLog::STATUS_EXITING,
|
||||||
);
|
);
|
||||||
default:
|
default:
|
||||||
throw new Exception("Unknown status '{$status}'!");
|
throw new Exception("Unknown status '{$status}'!");
|
||||||
|
|
|
@ -7,6 +7,7 @@ final class PhabricatorDaemonLog extends PhabricatorDaemonDAO
|
||||||
const STATUS_RUNNING = 'run';
|
const STATUS_RUNNING = 'run';
|
||||||
const STATUS_DEAD = 'dead';
|
const STATUS_DEAD = 'dead';
|
||||||
const STATUS_WAIT = 'wait';
|
const STATUS_WAIT = 'wait';
|
||||||
|
const STATUS_EXITING = 'exiting';
|
||||||
const STATUS_EXITED = 'exit';
|
const STATUS_EXITED = 'exit';
|
||||||
|
|
||||||
protected $daemon;
|
protected $daemon;
|
||||||
|
|
|
@ -17,8 +17,7 @@ final class PhabricatorDaemonLogListView extends AphrontView {
|
||||||
throw new Exception('Call setUser() before rendering!');
|
throw new Exception('Call setUser() before rendering!');
|
||||||
}
|
}
|
||||||
|
|
||||||
$list = id(new PHUIObjectItemListView())
|
$list = new PHUIObjectItemListView();
|
||||||
->setFlush(true);
|
|
||||||
foreach ($this->daemonLogs as $log) {
|
foreach ($this->daemonLogs as $log) {
|
||||||
$id = $log->getID();
|
$id = $log->getID();
|
||||||
$epoch = $log->getDateCreated();
|
$epoch = $log->getDateCreated();
|
||||||
|
@ -43,6 +42,10 @@ final class PhabricatorDaemonLogListView extends AphrontView {
|
||||||
'dead.'));
|
'dead.'));
|
||||||
$item->addIcon('fa-times grey', pht('Dead'));
|
$item->addIcon('fa-times grey', pht('Dead'));
|
||||||
break;
|
break;
|
||||||
|
case PhabricatorDaemonLog::STATUS_EXITING:
|
||||||
|
$item->addAttribute(pht('This daemon is exiting.'));
|
||||||
|
$item->addIcon('fa-check', pht('Exiting'));
|
||||||
|
break;
|
||||||
case PhabricatorDaemonLog::STATUS_EXITED:
|
case PhabricatorDaemonLog::STATUS_EXITED:
|
||||||
$item->setDisabled(true);
|
$item->setDisabled(true);
|
||||||
$item->addAttribute(pht('This daemon exited cleanly.'));
|
$item->addAttribute(pht('This daemon exited cleanly.'));
|
||||||
|
|
|
@ -8,7 +8,7 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon {
|
||||||
|
|
||||||
public function run() {
|
public function run() {
|
||||||
$this->setEngines(PhabricatorFactEngine::loadAllEngines());
|
$this->setEngines(PhabricatorFactEngine::loadAllEngines());
|
||||||
while (true) {
|
while (!$this->shouldExit()) {
|
||||||
$iterators = $this->getAllApplicationIterators();
|
$iterators = $this->getAllApplicationIterators();
|
||||||
foreach ($iterators as $iterator_name => $iterator) {
|
foreach ($iterators as $iterator_name => $iterator) {
|
||||||
$this->processIteratorWithCursor($iterator_name, $iterator);
|
$this->processIteratorWithCursor($iterator_name, $iterator);
|
||||||
|
|
|
@ -71,7 +71,7 @@ final class PhabricatorRepositoryPullLocalDaemon
|
||||||
$futures = array();
|
$futures = array();
|
||||||
$queue = array();
|
$queue = array();
|
||||||
|
|
||||||
while (true) {
|
while (!$this->shouldExit()) {
|
||||||
$pullable = $this->loadPullableRepositories($include, $exclude);
|
$pullable = $this->loadPullableRepositories($include, $exclude);
|
||||||
|
|
||||||
// If any repositories have the NEEDS_UPDATE flag set, pull them
|
// If any repositories have the NEEDS_UPDATE flag set, pull them
|
||||||
|
@ -422,6 +422,12 @@ final class PhabricatorRepositoryPullLocalDaemon
|
||||||
new PhutilNumber($sleep_duration)));
|
new PhutilNumber($sleep_duration)));
|
||||||
|
|
||||||
$this->sleep(1);
|
$this->sleep(1);
|
||||||
|
|
||||||
|
if ($this->shouldExit()) {
|
||||||
|
$this->log(pht('Awakened from sleep by graceful shutdown!'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if ($this->loadRepositoryUpdateMessages()) {
|
if ($this->loadRepositoryUpdateMessages()) {
|
||||||
$this->log(pht('Awakened from sleep by pending updates!'));
|
$this->log(pht('Awakened from sleep by pending updates!'));
|
||||||
break;
|
break;
|
||||||
|
|
|
@ -103,7 +103,7 @@ final class PhabricatorBot extends PhabricatorDaemon {
|
||||||
foreach ($this->handlers as $handler) {
|
foreach ($this->handlers as $handler) {
|
||||||
$handler->runBackgroundTasks();
|
$handler->runBackgroundTasks();
|
||||||
}
|
}
|
||||||
} while (true);
|
} while (!$this->shouldExit());
|
||||||
}
|
}
|
||||||
|
|
||||||
public function writeMessage(PhabricatorBotMessage $message) {
|
public function writeMessage(PhabricatorBotMessage $message) {
|
||||||
|
|
|
@ -31,7 +31,7 @@ final class PhabricatorGarbageCollectorDaemon extends PhabricatorDaemon {
|
||||||
// scans just to delete a handful of rows; wake up in a few hours.
|
// scans just to delete a handful of rows; wake up in a few hours.
|
||||||
$this->log(pht('All caught up, waiting for more garbage.'));
|
$this->log(pht('All caught up, waiting for more garbage.'));
|
||||||
$this->sleep(4 * (60 * 60));
|
$this->sleep(4 * (60 * 60));
|
||||||
} while (true);
|
} while (!$this->shouldExit());
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -40,7 +40,7 @@ final class PhabricatorTaskmasterDaemon extends PhabricatorDaemon {
|
||||||
}
|
}
|
||||||
|
|
||||||
$this->sleep($sleep);
|
$this->sleep($sleep);
|
||||||
} while (true);
|
} while (!$this->shouldExit());
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue