mirror of
https://we.phorge.it/source/phorge.git
synced 2024-11-09 16:32:39 +01:00
Save daemon state to database
Summary: To make it easier to monitor daemons, let's store their current state (running, died, exited, or unknown) to the db. The purpose of this is to provide more information on the daemon console about the status of daemons, especially when they are running on multiple machines. This is mostly backend work, with only a few frontend changes. (It is also dependent on a change to libphutil.) These changes will make dead or stuck daemons more obvious, and will allow more work on the frontend to hide daemons (and logs) that have exited cleanly, i.e. ones we don't care about any more. Test Plan: - run db migration, check in db that all daemons were marked as exited - start up a daemon, check in db that it is marked as running - open web interface, check that daemon is listed as running - after daemon has been running for a little bit, check in db that dateModified is being updated (indicating daemon is properly sending heartbeat) - kill -9 daemon (but don't run bin/phd yet), and check that db still shows it as running - edit daemon db entry to show it as being on a different host, and backdate dateModified field by 3 minutes, and check the web ui to show that the status is unknown. - change db entry to have proper host, check in web ui that daemon status is displayed as dead. Check db to see that the status was saved. - run bin/phd stop, and see that the formerly dead daemon is now exited. Reviewers: epriestley, vrana Reviewed By: epriestley CC: aran, Korvin Differential Revision: https://secure.phabricator.com/D3126
This commit is contained in:
parent
6e1f5e353a
commit
88caa45854
9 changed files with 155 additions and 26 deletions
4
resources/sql/patches/daemonstatus.sql
Normal file
4
resources/sql/patches/daemonstatus.sql
Normal file
|
@ -0,0 +1,4 @@
|
|||
ALTER TABLE {$NAMESPACE}_daemon.daemon_log
|
||||
ADD COLUMN `status` varchar(8) NOT NULL;
|
||||
|
||||
UPDATE {$NAMESPACE}_daemon.daemon_log SET `status`='exit';
|
|
@ -112,6 +112,7 @@ phutil_register_library_map(array(
|
|||
'ConduitAPI_conduit_ping_Method' => 'applications/conduit/method/conduit/ConduitAPI_conduit_ping_Method.php',
|
||||
'ConduitAPI_daemon_launched_Method' => 'applications/conduit/method/daemon/ConduitAPI_daemon_launched_Method.php',
|
||||
'ConduitAPI_daemon_log_Method' => 'applications/conduit/method/daemon/ConduitAPI_daemon_log_Method.php',
|
||||
'ConduitAPI_daemon_setstatus_Method' => 'applications/conduit/method/daemon/ConduitAPI_daemon_setstatus_Method.php',
|
||||
'ConduitAPI_differential_Method' => 'applications/conduit/method/differential/ConduitAPI_differential_Method.php',
|
||||
'ConduitAPI_differential_close_Method' => 'applications/conduit/method/differential/ConduitAPI_differential_close_Method.php',
|
||||
'ConduitAPI_differential_createcomment_Method' => 'applications/conduit/method/differential/ConduitAPI_differential_createcomment_Method.php',
|
||||
|
@ -1229,6 +1230,7 @@ phutil_register_library_map(array(
|
|||
'ConduitAPI_conduit_ping_Method' => 'ConduitAPIMethod',
|
||||
'ConduitAPI_daemon_launched_Method' => 'ConduitAPIMethod',
|
||||
'ConduitAPI_daemon_log_Method' => 'ConduitAPIMethod',
|
||||
'ConduitAPI_daemon_setstatus_Method' => 'ConduitAPIMethod',
|
||||
'ConduitAPI_differential_Method' => 'ConduitAPIMethod',
|
||||
'ConduitAPI_differential_close_Method' => 'ConduitAPIMethod',
|
||||
'ConduitAPI_differential_createcomment_Method' => 'ConduitAPIMethod',
|
||||
|
|
|
@ -58,6 +58,7 @@ final class ConduitAPI_daemon_launched_Method extends ConduitAPIMethod {
|
|||
$daemon_log->setDaemon($request->getValue('daemon'));
|
||||
$daemon_log->setHost($request->getValue('host'));
|
||||
$daemon_log->setPID($request->getValue('pid'));
|
||||
$daemon_log->setStatus(PhabricatorDaemonLog::STATUS_RUNNING);
|
||||
$daemon_log->setArgv(json_decode($request->getValue('argv')));
|
||||
|
||||
$daemon_log->save();
|
||||
|
|
|
@ -0,0 +1,66 @@
|
|||
<?php
|
||||
|
||||
/*
|
||||
* Copyright 2012 Facebook, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* @group conduit
|
||||
*/
|
||||
final class ConduitAPI_daemon_setstatus_Method extends ConduitAPIMethod {
|
||||
|
||||
public function shouldRequireAuthentication() {
|
||||
// TODO: Lock this down once we build phantoms.
|
||||
return false;
|
||||
}
|
||||
|
||||
public function shouldAllowUnguardedWrites() {
|
||||
return true;
|
||||
}
|
||||
|
||||
public function getMethodDescription() {
|
||||
return "Used by daemons to update their status.";
|
||||
}
|
||||
|
||||
public function defineParamTypes() {
|
||||
return array(
|
||||
'daemonLogID' => 'required string',
|
||||
'status' => 'required enum<unknown, run, timeout, dead, exit>',
|
||||
);
|
||||
}
|
||||
|
||||
public function defineReturnType() {
|
||||
return 'void';
|
||||
}
|
||||
|
||||
public function defineErrorTypes() {
|
||||
return array(
|
||||
'ERR-INVALID-ID' => 'An invalid daemonLogID was provided.',
|
||||
);
|
||||
}
|
||||
|
||||
protected function execute(ConduitAPIRequest $request) {
|
||||
|
||||
$daemon_log = id(new PhabricatorDaemonLog())
|
||||
->load($request->getValue('daemonLogID'));
|
||||
if (!$daemon_log) {
|
||||
throw new ConduitException('ERR-INVALID-ID');
|
||||
}
|
||||
$daemon_log->setStatus($request->getValue('status'));
|
||||
|
||||
$daemon_log->save();
|
||||
}
|
||||
|
||||
}
|
|
@ -42,38 +42,60 @@ final class PhabricatorDaemonLogListView extends AphrontView {
|
|||
foreach ($this->daemonLogs as $log) {
|
||||
$epoch = $log->getDateCreated();
|
||||
|
||||
if ($log->getHost() == php_uname('n')) {
|
||||
$status = $log->getStatus();
|
||||
if ($log->getHost() == php_uname('n') &&
|
||||
$status != PhabricatorDaemonLog::STATUS_EXITED &&
|
||||
$status != PhabricatorDaemonLog::STATUS_DEAD) {
|
||||
|
||||
$pid = $log->getPID();
|
||||
$is_running = PhabricatorDaemonReference::isProcessRunning($pid);
|
||||
|
||||
if ($is_running) {
|
||||
$running = phutil_render_tag(
|
||||
'span',
|
||||
array(
|
||||
'style' => 'color: #00cc00',
|
||||
'title' => 'Running',
|
||||
),
|
||||
'•');
|
||||
} else {
|
||||
$running = phutil_render_tag(
|
||||
'span',
|
||||
array(
|
||||
'style' => 'color: #cc0000',
|
||||
'title' => 'Not running',
|
||||
),
|
||||
'•');
|
||||
if (!$is_running) {
|
||||
$guard = AphrontWriteGuard::beginScopedUnguardedWrites();
|
||||
$log->setStatus(PhabricatorDaemonLog::STATUS_DEAD);
|
||||
$log->save();
|
||||
unset($guard);
|
||||
$status = PhabricatorDaemonLog::STATUS_DEAD;
|
||||
}
|
||||
} else {
|
||||
$running = phutil_render_tag(
|
||||
'span',
|
||||
array(
|
||||
'style' => 'color: #888888',
|
||||
'title' => 'Not on this host',
|
||||
),
|
||||
'?');
|
||||
}
|
||||
|
||||
$heartbeat_timeout =
|
||||
$log->getDateModified() + 3 * PhutilDaemonOverseer::HEARTBEAT_WAIT;
|
||||
if ($status == PhabricatorDaemonLog::STATUS_RUNNING &&
|
||||
$heartbeat_timeout < time()) {
|
||||
$status = PhabricatorDaemonLog::STATUS_UNKNOWN;
|
||||
}
|
||||
|
||||
switch ($status) {
|
||||
case PhabricatorDaemonLog::STATUS_RUNNING:
|
||||
$style = 'color: #00cc00';
|
||||
$title = 'Running';
|
||||
$symbol = '•';
|
||||
break;
|
||||
case PhabricatorDaemonLog::STATUS_DEAD:
|
||||
$style = 'color: #cc0000';
|
||||
$title = 'Died';
|
||||
$symbol = '•';
|
||||
break;
|
||||
case PhabricatorDaemonLog::STATUS_EXITED:
|
||||
$style = 'color: #000000';
|
||||
$title = 'Exited';
|
||||
$symbol = '•';
|
||||
break;
|
||||
case PhabricatorDaemonLog::STATUS_UNKNOWN:
|
||||
default: // fallthrough
|
||||
$style = 'color: #888888';
|
||||
$title = 'Unknown';
|
||||
$symbol = '?';
|
||||
}
|
||||
|
||||
$running = phutil_render_tag(
|
||||
'span',
|
||||
array(
|
||||
'style' => $style,
|
||||
'title' => $title,
|
||||
),
|
||||
$symbol);
|
||||
|
||||
$rows[] = array(
|
||||
$running,
|
||||
phutil_escape_html($log->getDaemon()),
|
||||
|
|
|
@ -53,6 +53,12 @@ final class PhabricatorDaemonControl {
|
|||
foreach ($daemons as $daemon) {
|
||||
$name = $daemon->getName();
|
||||
if (!$daemon->isRunning()) {
|
||||
$daemon_log = $daemon->loadDaemonLog();
|
||||
if ($daemon_log) {
|
||||
$daemon_log->setStatus(PhabricatorDaemonLog::STATUS_DEAD);
|
||||
$daemon_log->save();
|
||||
}
|
||||
|
||||
$status = 2;
|
||||
$name = '<DEAD> '.$name;
|
||||
}
|
||||
|
@ -110,6 +116,11 @@ final class PhabricatorDaemonControl {
|
|||
if (!$daemon->isRunning()) {
|
||||
echo "Daemon is not running.\n";
|
||||
unset($running[$key]);
|
||||
$daemon_log = $daemon->loadDaemonLog();
|
||||
if ($daemon_log) {
|
||||
$daemon_log->setStatus(PhabricatorDaemonLog::STATUS_EXITED);
|
||||
$daemon_log->save();
|
||||
}
|
||||
} else {
|
||||
posix_kill($pid, SIGINT);
|
||||
}
|
||||
|
|
|
@ -23,6 +23,8 @@ final class PhabricatorDaemonReference {
|
|||
private $start;
|
||||
private $pidFile;
|
||||
|
||||
private $daemonLog;
|
||||
|
||||
public static function newFromDictionary(array $dict) {
|
||||
$ref = new PhabricatorDaemonReference();
|
||||
|
||||
|
@ -33,6 +35,17 @@ final class PhabricatorDaemonReference {
|
|||
return $ref;
|
||||
}
|
||||
|
||||
public function loadDaemonLog() {
|
||||
if (!$this->daemonLog) {
|
||||
$this->daemonLog = id(new PhabricatorDaemonLog())->loadOneWhere(
|
||||
'daemon = %s AND pid = %d AND dateCreated = %d',
|
||||
$this->name,
|
||||
$this->pid,
|
||||
$this->start);
|
||||
}
|
||||
return $this->daemonLog;
|
||||
}
|
||||
|
||||
public function getPID() {
|
||||
return $this->pid;
|
||||
}
|
||||
|
|
|
@ -18,10 +18,16 @@
|
|||
|
||||
final class PhabricatorDaemonLog extends PhabricatorDaemonDAO {
|
||||
|
||||
const STATUS_UNKNOWN = 'unknown';
|
||||
const STATUS_RUNNING = 'run';
|
||||
const STATUS_DEAD = 'dead';
|
||||
const STATUS_EXITED = 'exit';
|
||||
|
||||
protected $daemon;
|
||||
protected $host;
|
||||
protected $pid;
|
||||
protected $argv;
|
||||
protected $status;
|
||||
|
||||
public function getConfiguration() {
|
||||
return array(
|
||||
|
|
|
@ -932,6 +932,10 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList {
|
|||
'type' => 'php',
|
||||
'name' => $this->getPatchPath('migrate-maniphest-revisions.php'),
|
||||
),
|
||||
'daemonstatus.sql' => array(
|
||||
'type' => 'sql',
|
||||
'name' => $this->getPatchPath('daemonstatus.sql'),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue