1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-30 02:32:42 +01:00

Send graceful shutdown signals to daemons in Phabricator

Summary:
Fixes T5855. Adds a `--graceful N` flag to `phd stop` and `phd restart`.

`phd` will send SIGINT, wait `N` seconds, SIGTERM, wait 15 seconds, and SIGKILL. By default, `N` is 15.

Test Plan:
  - Ran `bin/phd debug ...` and used `^C` to interrupt daemons. Saw graceful shutdown behavior, and abrupt termination on multiple `^C`.
  - Ran `bin/phd start`, `bin/phd stop` and `bin/phd restart` with `--graceful` set to various things, notably `0`. Saw graceful shutdowns on the CLI and in the web UI. With `0`, abrupt shutdowns.

Reviewers: btrahan, hach-que

Reviewed By: hach-que

Subscribers: epriestley

Maniphest Tasks: T5855

Differential Revision: https://secure.phabricator.com/D10228
This commit is contained in:
epriestley 2014-08-11 20:18:31 -07:00
parent aab0ed1c50
commit 9309723ac4
13 changed files with 117 additions and 41 deletions

View file

@ -53,6 +53,10 @@ final class PhabricatorDaemonLogViewController
$tag->setBackgroundColor(PHUITagView::COLOR_BLUE); $tag->setBackgroundColor(PHUITagView::COLOR_BLUE);
$tag->setName(pht('Waiting')); $tag->setName(pht('Waiting'));
break; break;
case PhabricatorDaemonLog::STATUS_EXITING:
$tag->setBackgroundColor(PHUITagView::COLOR_YELLOW);
$tag->setName(pht('Exiting'));
break;
case PhabricatorDaemonLog::STATUS_EXITED: case PhabricatorDaemonLog::STATUS_EXITED:
$tag->setBackgroundColor(PHUITagView::COLOR_GREY); $tag->setBackgroundColor(PHUITagView::COLOR_GREY);
$tag->setName(pht('Exited')); $tag->setName(pht('Exited'));
@ -136,6 +140,10 @@ final class PhabricatorDaemonLogViewController
phutil_format_relative_time($unknown_time), phutil_format_relative_time($unknown_time),
phutil_format_relative_time($wait_time)); phutil_format_relative_time($wait_time));
break; break;
case PhabricatorDaemonLog::STATUS_EXITING:
$details = pht(
'This daemon is shutting down gracefully.');
break;
case PhabricatorDaemonLog::STATUS_EXITED: case PhabricatorDaemonLog::STATUS_EXITED:
$details = pht( $details = pht(
'This daemon exited normally and is no longer running.'); 'This daemon exited normally and is no longer running.');

View file

@ -8,6 +8,7 @@ final class PhabricatorDaemonEventListener extends PhabricatorEventListener {
$this->listen(PhutilDaemonOverseer::EVENT_DID_LAUNCH); $this->listen(PhutilDaemonOverseer::EVENT_DID_LAUNCH);
$this->listen(PhutilDaemonOverseer::EVENT_DID_LOG); $this->listen(PhutilDaemonOverseer::EVENT_DID_LOG);
$this->listen(PhutilDaemonOverseer::EVENT_DID_HEARTBEAT); $this->listen(PhutilDaemonOverseer::EVENT_DID_HEARTBEAT);
$this->listen(PhutilDaemonOverseer::EVENT_WILL_GRACEFUL);
$this->listen(PhutilDaemonOverseer::EVENT_WILL_EXIT); $this->listen(PhutilDaemonOverseer::EVENT_WILL_EXIT);
} }
@ -22,6 +23,9 @@ final class PhabricatorDaemonEventListener extends PhabricatorEventListener {
case PhutilDaemonOverseer::EVENT_DID_LOG: case PhutilDaemonOverseer::EVENT_DID_LOG:
$this->handleLogEvent($event); $this->handleLogEvent($event);
break; break;
case PhutilDaemonOverseer::EVENT_WILL_GRACEFUL:
$this->handleGracefulEvent($event);
break;
case PhutilDaemonOverseer::EVENT_WILL_EXIT: case PhutilDaemonOverseer::EVENT_WILL_EXIT:
$this->handleExitEvent($event); $this->handleExitEvent($event);
break; break;
@ -86,6 +90,13 @@ final class PhabricatorDaemonEventListener extends PhabricatorEventListener {
} }
} }
private function handleGracefulEvent(PhutilEvent $event) {
$id = $event->getValue('id');
$daemon = $this->getDaemon($id);
$daemon->setStatus(PhabricatorDaemonLog::STATUS_EXITING)->save();
}
private function handleExitEvent(PhutilEvent $event) { private function handleExitEvent(PhutilEvent $event) {
$id = $event->getValue('id'); $id = $event->getValue('id');

View file

@ -9,11 +9,22 @@ final class PhabricatorDaemonManagementRestartWorkflow
->setSynopsis( ->setSynopsis(
pht( pht(
'Stop, then start the standard daemon loadout.')) 'Stop, then start the standard daemon loadout.'))
->setArguments(array()); ->setArguments(
array(
array(
'name' => 'graceful',
'param' => 'seconds',
'help' => pht(
'Grace period for daemons to attempt a clean shutdown, in '.
'seconds. Defaults to __15__ seconds.'),
'default' => 15,
),
));
} }
public function execute(PhutilArgumentParser $args) { public function execute(PhutilArgumentParser $args) {
$err = $this->executeStopCommand(array()); $graceful = $args->getArg('graceful');
$err = $this->executeStopCommand(array(), $graceful);
if ($err) { if ($err) {
return $err; return $err;
} }

View file

@ -12,6 +12,14 @@ final class PhabricatorDaemonManagementStopWorkflow
'Use **phd status** to find PIDs.')) 'Use **phd status** to find PIDs.'))
->setArguments( ->setArguments(
array( array(
array(
'name' => 'graceful',
'param' => 'seconds',
'help' => pht(
'Grace period for daemons to attempt a clean shutdown, in '.
'seconds. Defaults to __15__ seconds.'),
'default' => 15,
),
array( array(
'name' => 'pids', 'name' => 'pids',
'wildcard' => true, 'wildcard' => true,
@ -21,7 +29,8 @@ final class PhabricatorDaemonManagementStopWorkflow
public function execute(PhutilArgumentParser $args) { public function execute(PhutilArgumentParser $args) {
$pids = $args->getArg('pids'); $pids = $args->getArg('pids');
return $this->executeStopCommand($pids); $graceful = $args->getArg('graceful');
return $this->executeStopCommand($pids, $graceful);
} }
} }

View file

@ -286,7 +286,7 @@ abstract class PhabricatorDaemonManagementWorkflow
return 0; return 0;
} }
protected final function executeStopCommand(array $pids) { protected final function executeStopCommand(array $pids, $grace_period) {
$console = PhutilConsole::getConsole(); $console = PhutilConsole::getConsole();
$daemons = $this->loadRunningDaemons(); $daemons = $this->loadRunningDaemons();
@ -325,39 +325,20 @@ abstract class PhabricatorDaemonManagementWorkflow
} }
$all_daemons = $running; $all_daemons = $running;
foreach ($running as $key => $daemon) {
$pid = $daemon->getPID();
$name = $daemon->getName();
$console->writeErr(pht("Stopping daemon '%s' (%s)...", $name, $pid)."\n"); // If we're doing a graceful shutdown, try SIGINT first.
if (!$daemon->isRunning()) { if ($grace_period) {
$console->writeErr(pht('Daemon is not running.')."\n"); $running = $this->sendSignal($running, SIGINT, $grace_period);
unset($running[$key]);
$daemon->updateStatus(PhabricatorDaemonLog::STATUS_EXITED);
} else {
posix_kill($pid, SIGINT);
}
} }
$start = time(); // If we still have daemons, SIGTERM them.
do { if ($running) {
foreach ($running as $key => $daemon) { $running = $this->sendSignal($running, SIGTERM, 15);
$pid = $daemon->getPID(); }
if (!$daemon->isRunning()) {
$console->writeOut(pht('Daemon %s exited normally.', $pid)."\n");
unset($running[$key]);
}
}
if (empty($running)) {
break;
}
usleep(100000);
} while (time() < $start + 15);
foreach ($running as $key => $daemon) { // If the overseer is still alive, SIGKILL it.
$pid = $daemon->getPID(); if ($running) {
$console->writeErr(pht('Sending daemon %s a SIGKILL.', $pid)."\n"); $this->sendSignal($running, SIGKILL, 0);
posix_kill($pid, SIGKILL);
} }
foreach ($all_daemons as $daemon) { foreach ($all_daemons as $daemon) {
@ -369,6 +350,49 @@ abstract class PhabricatorDaemonManagementWorkflow
return 0; return 0;
} }
private function sendSignal(array $daemons, $signo, $wait) {
$console = PhutilConsole::getConsole();
foreach ($daemons as $key => $daemon) {
$pid = $daemon->getPID();
$name = $daemon->getName();
switch ($signo) {
case SIGINT:
$message = pht("Interrupting daemon '%s' (%s)...", $name, $pid);
break;
case SIGTERM:
$message = pht("Terminating daemon '%s' (%s)...", $name, $pid);
break;
case SIGKILL:
$message = pht("Killing daemon '%s' (%s)...", $name, $pid);
break;
}
$console->writeOut("%s\n", $message);
posix_kill($pid, $signo);
}
if ($wait) {
$start = PhabricatorTime::getNow();
do {
foreach ($daemons as $key => $daemon) {
$pid = $daemon->getPID();
if (!$daemon->isRunning()) {
$console->writeOut(pht('Daemon %s exited.', $pid)."\n");
unset($daemons[$key]);
}
}
if (empty($daemons)) {
break;
}
usleep(100000);
} while (PhabricatorTime::getNow() < $start + $wait);
}
return $daemons;
}
private function freeActiveLeases() { private function freeActiveLeases() {
$task_table = id(new PhabricatorWorkerActiveTask()); $task_table = id(new PhabricatorWorkerActiveTask());
$conn_w = $task_table->establishConnection('w'); $conn_w = $task_table->establishConnection('w');

View file

@ -67,6 +67,7 @@ final class PhabricatorDaemonLogQuery
$status_running = PhabricatorDaemonLog::STATUS_RUNNING; $status_running = PhabricatorDaemonLog::STATUS_RUNNING;
$status_unknown = PhabricatorDaemonLog::STATUS_UNKNOWN; $status_unknown = PhabricatorDaemonLog::STATUS_UNKNOWN;
$status_wait = PhabricatorDaemonLog::STATUS_WAIT; $status_wait = PhabricatorDaemonLog::STATUS_WAIT;
$status_exiting = PhabricatorDaemonLog::STATUS_EXITING;
$status_exited = PhabricatorDaemonLog::STATUS_EXITED; $status_exited = PhabricatorDaemonLog::STATUS_EXITED;
$status_dead = PhabricatorDaemonLog::STATUS_DEAD; $status_dead = PhabricatorDaemonLog::STATUS_DEAD;
@ -77,7 +78,8 @@ final class PhabricatorDaemonLogQuery
$seen = $daemon->getDateModified(); $seen = $daemon->getDateModified();
$is_running = ($status == $status_running) || $is_running = ($status == $status_running) ||
($status == $status_wait); ($status == $status_wait) ||
($status == $status_exiting);
// If we haven't seen the daemon recently, downgrade its status to // If we haven't seen the daemon recently, downgrade its status to
// unknown. // unknown.
@ -160,6 +162,7 @@ final class PhabricatorDaemonLogQuery
PhabricatorDaemonLog::STATUS_UNKNOWN, PhabricatorDaemonLog::STATUS_UNKNOWN,
PhabricatorDaemonLog::STATUS_RUNNING, PhabricatorDaemonLog::STATUS_RUNNING,
PhabricatorDaemonLog::STATUS_WAIT, PhabricatorDaemonLog::STATUS_WAIT,
PhabricatorDaemonLog::STATUS_EXITING,
); );
default: default:
throw new Exception("Unknown status '{$status}'!"); throw new Exception("Unknown status '{$status}'!");

View file

@ -7,6 +7,7 @@ final class PhabricatorDaemonLog extends PhabricatorDaemonDAO
const STATUS_RUNNING = 'run'; const STATUS_RUNNING = 'run';
const STATUS_DEAD = 'dead'; const STATUS_DEAD = 'dead';
const STATUS_WAIT = 'wait'; const STATUS_WAIT = 'wait';
const STATUS_EXITING = 'exiting';
const STATUS_EXITED = 'exit'; const STATUS_EXITED = 'exit';
protected $daemon; protected $daemon;

View file

@ -17,8 +17,7 @@ final class PhabricatorDaemonLogListView extends AphrontView {
throw new Exception('Call setUser() before rendering!'); throw new Exception('Call setUser() before rendering!');
} }
$list = id(new PHUIObjectItemListView()) $list = new PHUIObjectItemListView();
->setFlush(true);
foreach ($this->daemonLogs as $log) { foreach ($this->daemonLogs as $log) {
$id = $log->getID(); $id = $log->getID();
$epoch = $log->getDateCreated(); $epoch = $log->getDateCreated();
@ -43,6 +42,10 @@ final class PhabricatorDaemonLogListView extends AphrontView {
'dead.')); 'dead.'));
$item->addIcon('fa-times grey', pht('Dead')); $item->addIcon('fa-times grey', pht('Dead'));
break; break;
case PhabricatorDaemonLog::STATUS_EXITING:
$item->addAttribute(pht('This daemon is exiting.'));
$item->addIcon('fa-check', pht('Exiting'));
break;
case PhabricatorDaemonLog::STATUS_EXITED: case PhabricatorDaemonLog::STATUS_EXITED:
$item->setDisabled(true); $item->setDisabled(true);
$item->addAttribute(pht('This daemon exited cleanly.')); $item->addAttribute(pht('This daemon exited cleanly.'));

View file

@ -8,7 +8,7 @@ final class PhabricatorFactDaemon extends PhabricatorDaemon {
public function run() { public function run() {
$this->setEngines(PhabricatorFactEngine::loadAllEngines()); $this->setEngines(PhabricatorFactEngine::loadAllEngines());
while (true) { while (!$this->shouldExit()) {
$iterators = $this->getAllApplicationIterators(); $iterators = $this->getAllApplicationIterators();
foreach ($iterators as $iterator_name => $iterator) { foreach ($iterators as $iterator_name => $iterator) {
$this->processIteratorWithCursor($iterator_name, $iterator); $this->processIteratorWithCursor($iterator_name, $iterator);

View file

@ -71,7 +71,7 @@ final class PhabricatorRepositoryPullLocalDaemon
$futures = array(); $futures = array();
$queue = array(); $queue = array();
while (true) { while (!$this->shouldExit()) {
$pullable = $this->loadPullableRepositories($include, $exclude); $pullable = $this->loadPullableRepositories($include, $exclude);
// If any repositories have the NEEDS_UPDATE flag set, pull them // If any repositories have the NEEDS_UPDATE flag set, pull them
@ -422,6 +422,12 @@ final class PhabricatorRepositoryPullLocalDaemon
new PhutilNumber($sleep_duration))); new PhutilNumber($sleep_duration)));
$this->sleep(1); $this->sleep(1);
if ($this->shouldExit()) {
$this->log(pht('Awakened from sleep by graceful shutdown!'));
return;
}
if ($this->loadRepositoryUpdateMessages()) { if ($this->loadRepositoryUpdateMessages()) {
$this->log(pht('Awakened from sleep by pending updates!')); $this->log(pht('Awakened from sleep by pending updates!'));
break; break;

View file

@ -103,7 +103,7 @@ final class PhabricatorBot extends PhabricatorDaemon {
foreach ($this->handlers as $handler) { foreach ($this->handlers as $handler) {
$handler->runBackgroundTasks(); $handler->runBackgroundTasks();
} }
} while (true); } while (!$this->shouldExit());
} }
public function writeMessage(PhabricatorBotMessage $message) { public function writeMessage(PhabricatorBotMessage $message) {

View file

@ -31,7 +31,7 @@ final class PhabricatorGarbageCollectorDaemon extends PhabricatorDaemon {
// scans just to delete a handful of rows; wake up in a few hours. // scans just to delete a handful of rows; wake up in a few hours.
$this->log(pht('All caught up, waiting for more garbage.')); $this->log(pht('All caught up, waiting for more garbage.'));
$this->sleep(4 * (60 * 60)); $this->sleep(4 * (60 * 60));
} while (true); } while (!$this->shouldExit());
} }

View file

@ -40,7 +40,7 @@ final class PhabricatorTaskmasterDaemon extends PhabricatorDaemon {
} }
$this->sleep($sleep); $this->sleep($sleep);
} while (true); } while (!$this->shouldExit());
} }
} }