mirror of
https://we.phorge.it/source/phorge.git
synced 2024-11-12 18:02:40 +01:00
Add an option to make it easier to debug page hangs
Summary: Fixes T6044. We've had two cases (both the same install, coincidentally) where pages got hung doing too much data fetching. When pages hang, we don't get a useful stack trace out of them, since nginx, php-fpm, or PHP eventually terminates things in a non-useful way without any diagnostic information. The second time (the recent Macros issue) I was able to walk the install through removing limits on nginx, php-fpm, php, and eventually getting a profile by letting the page run for several minutes until the request completed. However, this install is exceptionally technically proficient and this was still a big pain for everyone, and this approach would not have worked if the page actually looped rather than just taking a long time. Provide `debug.time-limit`, which should give us a better tool for reacting to this situation: by setting it to a small value (like 10), we'll kill the page after 10 seconds with a trace, before nginx/php-fpm/php/etc can kill it uselessly. Hopefully that will be enough information to find the issue (generally, getting a trace has been 95% of the problem in the two cases we've encountered). Test Plan: Set this option to `3` and added a sleep loop, saw a termination after 3 seconds with a useful trace. Reviewers: btrahan Reviewed By: btrahan Subscribers: csilvers, joshuaspence, epriestley Maniphest Tasks: T6044 Differential Revision: https://secure.phabricator.com/D10465
This commit is contained in:
parent
d3cd9115f9
commit
cae59d8345
3 changed files with 96 additions and 0 deletions
|
@ -46,6 +46,31 @@ final class PhabricatorDeveloperConfigOptions
|
||||||
"enable this option in production.\n\n".
|
"enable this option in production.\n\n".
|
||||||
"You must enable DarkConsole by setting {{darkconsole.enabled}} ".
|
"You must enable DarkConsole by setting {{darkconsole.enabled}} ".
|
||||||
"before this option will have any effect.")),
|
"before this option will have any effect.")),
|
||||||
|
$this->newOption('debug.time-limit', 'int', null)
|
||||||
|
->setSummary(
|
||||||
|
pht(
|
||||||
|
'Limit page execution time to debug hangs.'))
|
||||||
|
->setDescription(
|
||||||
|
pht(
|
||||||
|
"This option can help debug pages which are taking a very ".
|
||||||
|
"long time (more than 30 seconds) to render.\n\n".
|
||||||
|
"If a page is slow to render (but taking less than 30 seconds), ".
|
||||||
|
"the best tools to use to figure out why it is slow are usually ".
|
||||||
|
"the DarkConsole service call profiler and XHProf.\n\n".
|
||||||
|
"However, if a request takes a very long time to return, some ".
|
||||||
|
"components (like Apache, nginx, or PHP itself) may abort the ".
|
||||||
|
"request before it finishes. This can prevent you from using ".
|
||||||
|
"profiling tools to understand page performance in detail.\n\n".
|
||||||
|
"In these cases, you can use this option to force the page to ".
|
||||||
|
"abort after a smaller number of seconds (for example, 10), and ".
|
||||||
|
"dump a useful stack trace. This can provide useful information ".
|
||||||
|
"about why a page is hanging.\n\n".
|
||||||
|
"To use this option, set it to a small number (like 10), and ".
|
||||||
|
"reload a hanging page. The page should exit after 10 seconds ".
|
||||||
|
"and give you a stack trace.\n\n".
|
||||||
|
"You should turn this option off (set it to 0) when you are ".
|
||||||
|
"done with it. Leaving it on creates a small amount of overhead ".
|
||||||
|
"for all requests, even if they do not hit the time limit.")),
|
||||||
$this->newOption('debug.stop-on-redirect', 'bool', false)
|
$this->newOption('debug.stop-on-redirect', 'bool', false)
|
||||||
->setBoolOptions(
|
->setBoolOptions(
|
||||||
array(
|
array(
|
||||||
|
|
|
@ -38,6 +38,7 @@
|
||||||
final class PhabricatorStartup {
|
final class PhabricatorStartup {
|
||||||
|
|
||||||
private static $startTime;
|
private static $startTime;
|
||||||
|
private static $debugTimeLimit;
|
||||||
private static $globals = array();
|
private static $globals = array();
|
||||||
private static $capturingOutput;
|
private static $capturingOutput;
|
||||||
private static $rawInput;
|
private static $rawInput;
|
||||||
|
@ -226,6 +227,70 @@ final class PhabricatorStartup {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/* -( Debug Time Limit )--------------------------------------------------- */
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set a time limit (in seconds) for the current script. After time expires,
|
||||||
|
* the script fatals.
|
||||||
|
*
|
||||||
|
* This works like `max_execution_time`, but prints out a useful stack trace
|
||||||
|
* when the time limit expires. This is primarily intended to make it easier
|
||||||
|
* to debug pages which hang by allowing extraction of a stack trace: set a
|
||||||
|
* short debug limit, then use the trace to figure out what's happening.
|
||||||
|
*
|
||||||
|
* The limit is implemented with a tick function, so enabling it implies
|
||||||
|
* some accounting overhead.
|
||||||
|
*
|
||||||
|
* @param int Time limit in seconds.
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
public static function setDebugTimeLimit($limit) {
|
||||||
|
self::$debugTimeLimit = $limit;
|
||||||
|
|
||||||
|
static $initialized;
|
||||||
|
if (!$initialized) {
|
||||||
|
declare(ticks=1);
|
||||||
|
register_tick_function(array('PhabricatorStartup', 'onDebugTick'));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Callback tick function used by @{method:setDebugTimeLimit}.
|
||||||
|
*
|
||||||
|
* Fatals with a useful stack trace after the time limit expires.
|
||||||
|
*
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
public static function onDebugTick() {
|
||||||
|
$limit = self::$debugTimeLimit;
|
||||||
|
if (!$limit) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
$elapsed = (microtime(true) - self::getStartTime());
|
||||||
|
if ($elapsed > $limit) {
|
||||||
|
$frames = array();
|
||||||
|
foreach (debug_backtrace() as $frame) {
|
||||||
|
$file = isset($frame['file']) ? $frame['file'] : '-';
|
||||||
|
$file = basename($file);
|
||||||
|
|
||||||
|
$line = isset($frame['line']) ? $frame['line'] : '-';
|
||||||
|
$class = isset($frame['class']) ? $frame['class'].'->' : null;
|
||||||
|
$func = isset($frame['function']) ? $frame['function'].'()' : '?';
|
||||||
|
|
||||||
|
$frames[] = "{$file}:{$line} {$class}{$func}";
|
||||||
|
}
|
||||||
|
|
||||||
|
self::didFatal(
|
||||||
|
"Request aborted by debug time limit after {$limit} seconds.\n\n".
|
||||||
|
"STACK TRACE\n".
|
||||||
|
implode("\n", $frames));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/* -( In Case of Apocalypse )---------------------------------------------- */
|
/* -( In Case of Apocalypse )---------------------------------------------- */
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -16,6 +16,12 @@ try {
|
||||||
PhabricatorStartup::loadCoreLibraries();
|
PhabricatorStartup::loadCoreLibraries();
|
||||||
|
|
||||||
PhabricatorEnv::initializeWebEnvironment();
|
PhabricatorEnv::initializeWebEnvironment();
|
||||||
|
|
||||||
|
$debug_time_limit = PhabricatorEnv::getEnvConfig('debug.time-limit');
|
||||||
|
if ($debug_time_limit) {
|
||||||
|
PhabricatorStartup::setDebugTimeLimit($debug_time_limit);
|
||||||
|
}
|
||||||
|
|
||||||
$show_unexpected_traces = PhabricatorEnv::getEnvConfig(
|
$show_unexpected_traces = PhabricatorEnv::getEnvConfig(
|
||||||
'phabricator.developer-mode');
|
'phabricator.developer-mode');
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue