1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-18 12:52:42 +01:00

Skeleton for "Multimeter", a performance sampling application

Summary:
Ref T6930. This application collects and displays performance samples -- roughly, things Phabricator spent some kind of resource on. It will collect samples on different types of resources and events:

  - Wall time (queries, service calls, pages)
  - Bytes In / Bytes Out (requests)
  - Implicit requests to CSS/JS (static resources)

I've started with the simplest case (static resources), since this can be used in an immediate, straghtforward way to improve packaging (look at which individual files have the most requests recently).

There's no aggregation yet and a lot of the data isn't collected properly. Future diffs will add more dimension data (controllers, users), more event and resource types (queries, service calls, wall time), and more display options (aggregation, sorting).

Test Plan: {F389344}

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T6930

Differential Revision: https://secure.phabricator.com/D12623
This commit is contained in:
epriestley 2015-05-01 13:19:43 -07:00
parent d25245414c
commit 7b6c320e15
22 changed files with 641 additions and 0 deletions

View file

@ -0,0 +1,14 @@
CREATE TABLE {$NAMESPACE}_multimeter.multimeter_event (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
eventType INT UNSIGNED NOT NULL,
eventLabelID INT UNSIGNED NOT NULL,
resourceCost BIGINT NOT NULL,
sampleRate INT UNSIGNED NOT NULL,
eventContextID INT UNSIGNED NOT NULL,
eventHostID INT UNSIGNED NOT NULL,
eventViewerID INT UNSIGNED NOT NULL,
epoch INT UNSIGNED NOT NULL,
requestKey BINARY(12) NOT NULL,
KEY `key_request` (requestKey),
KEY `key_type` (eventType, epoch)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

View file

@ -0,0 +1,6 @@
CREATE TABLE {$NAMESPACE}_multimeter.multimeter_host (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
name LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL,
nameHash BINARY(12) NOT NULL,
UNIQUE KEY `key_hash` (nameHash)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

View file

@ -0,0 +1,6 @@
CREATE TABLE {$NAMESPACE}_multimeter.multimeter_viewer (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
name LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL,
nameHash BINARY(12) NOT NULL,
UNIQUE KEY `key_hash` (nameHash)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

View file

@ -0,0 +1,6 @@
CREATE TABLE {$NAMESPACE}_multimeter.multimeter_context (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
name LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL,
nameHash BINARY(12) NOT NULL,
UNIQUE KEY `key_hash` (nameHash)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

View file

@ -0,0 +1,6 @@
CREATE TABLE {$NAMESPACE}_multimeter.multimeter_label (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
name LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL,
nameHash BINARY(12) NOT NULL,
UNIQUE KEY `key_hash` (nameHash)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

View file

@ -1080,6 +1080,17 @@ phutil_register_library_map(array(
'MetaMTAMailSentGarbageCollector' => 'applications/metamta/garbagecollector/MetaMTAMailSentGarbageCollector.php',
'MetaMTANotificationType' => 'applications/metamta/constants/MetaMTANotificationType.php',
'MetaMTAReceivedMailStatus' => 'applications/metamta/constants/MetaMTAReceivedMailStatus.php',
'MultimeterContext' => 'applications/multimeter/storage/MultimeterContext.php',
'MultimeterControl' => 'applications/multimeter/data/MultimeterControl.php',
'MultimeterController' => 'applications/multimeter/controller/MultimeterController.php',
'MultimeterDAO' => 'applications/multimeter/storage/MultimeterDAO.php',
'MultimeterDimension' => 'applications/multimeter/storage/MultimeterDimension.php',
'MultimeterEvent' => 'applications/multimeter/storage/MultimeterEvent.php',
'MultimeterEventGarbageCollector' => 'applications/multimeter/garbagecollector/MultimeterEventGarbageCollector.php',
'MultimeterHost' => 'applications/multimeter/storage/MultimeterHost.php',
'MultimeterLabel' => 'applications/multimeter/storage/MultimeterLabel.php',
'MultimeterSampleController' => 'applications/multimeter/controller/MultimeterSampleController.php',
'MultimeterViewer' => 'applications/multimeter/storage/MultimeterViewer.php',
'NuanceConduitAPIMethod' => 'applications/nuance/conduit/NuanceConduitAPIMethod.php',
'NuanceController' => 'applications/nuance/controller/NuanceController.php',
'NuanceCreateItemConduitAPIMethod' => 'applications/nuance/conduit/NuanceCreateItemConduitAPIMethod.php',
@ -2062,6 +2073,7 @@ phutil_register_library_map(array(
'PhabricatorMetronomicTriggerClock' => 'infrastructure/daemon/workers/clock/PhabricatorMetronomicTriggerClock.php',
'PhabricatorMultiColumnUIExample' => 'applications/uiexample/examples/PhabricatorMultiColumnUIExample.php',
'PhabricatorMultiFactorSettingsPanel' => 'applications/settings/panel/PhabricatorMultiFactorSettingsPanel.php',
'PhabricatorMultimeterApplication' => 'applications/multimeter/application/PhabricatorMultimeterApplication.php',
'PhabricatorMustVerifyEmailController' => 'applications/auth/controller/PhabricatorMustVerifyEmailController.php',
'PhabricatorMySQLConfigOptions' => 'applications/config/option/PhabricatorMySQLConfigOptions.php',
'PhabricatorMySQLFileStorageEngine' => 'applications/files/engine/PhabricatorMySQLFileStorageEngine.php',
@ -4381,6 +4393,16 @@ phutil_register_library_map(array(
'MetaMTAMailSentGarbageCollector' => 'PhabricatorGarbageCollector',
'MetaMTANotificationType' => 'MetaMTAConstants',
'MetaMTAReceivedMailStatus' => 'MetaMTAConstants',
'MultimeterContext' => 'MultimeterDimension',
'MultimeterController' => 'PhabricatorController',
'MultimeterDAO' => 'PhabricatorLiskDAO',
'MultimeterDimension' => 'MultimeterDAO',
'MultimeterEvent' => 'MultimeterDAO',
'MultimeterEventGarbageCollector' => 'PhabricatorGarbageCollector',
'MultimeterHost' => 'MultimeterDimension',
'MultimeterLabel' => 'MultimeterDimension',
'MultimeterSampleController' => 'MultimeterController',
'MultimeterViewer' => 'MultimeterDimension',
'NuanceConduitAPIMethod' => 'ConduitAPIMethod',
'NuanceController' => 'PhabricatorController',
'NuanceCreateItemConduitAPIMethod' => 'NuanceConduitAPIMethod',
@ -5443,6 +5465,7 @@ phutil_register_library_map(array(
'PhabricatorMetronomicTriggerClock' => 'PhabricatorTriggerClock',
'PhabricatorMultiColumnUIExample' => 'PhabricatorUIExample',
'PhabricatorMultiFactorSettingsPanel' => 'PhabricatorSettingsPanel',
'PhabricatorMultimeterApplication' => 'PhabricatorApplication',
'PhabricatorMustVerifyEmailController' => 'PhabricatorAuthController',
'PhabricatorMySQLConfigOptions' => 'PhabricatorApplicationConfigOptions',
'PhabricatorMySQLFileStorageEngine' => 'PhabricatorFileStorageEngine',

View file

@ -58,8 +58,15 @@ abstract class AphrontApplicationConfiguration {
* @phutil-external-symbol class PhabricatorStartup
*/
public static function runHTTPRequest(AphrontHTTPSink $sink) {
$multimeter = MultimeterControl::newInstance();
$multimeter->setEventContext('<http-init>');
$multimeter->setEventViewer('<none>');
PhabricatorEnv::initializeWebEnvironment();
$multimeter->setSampleRate(
PhabricatorEnv::getEnvConfig('debug.sample-rate'));
$debug_time_limit = PhabricatorEnv::getEnvConfig('debug.time-limit');
if ($debug_time_limit) {
PhabricatorStartup::setDebugTimeLimit($debug_time_limit);
@ -135,6 +142,8 @@ abstract class AphrontApplicationConfiguration {
$access_log->write();
$multimeter->saveEvents();
DarkConsoleXHProfPluginAPI::saveProfilerSample($access_log);
// Add points to the rate limits for this request.

View file

@ -144,6 +144,9 @@ final class CelerityStaticResourceResponse {
$uri = $this->getURI($map, $name);
$type = $map->getResourceTypeForName($name);
$event_type = MultimeterEvent::TYPE_STATIC_RESOURCE;
MultimeterControl::getInstance()->newEvent($event_type, 'rsrc.'.$name, 1);
switch ($type) {
case 'css':
return phutil_tag(

View file

@ -118,6 +118,27 @@ final class PhabricatorDeveloperConfigOptions
"data to look at eventually). In development, it may be useful to ".
"set it to 1 in order to debug performance problems.\n\n".
"NOTE: You must install XHProf for profiling to work.")),
$this->newOption('debug.sample-rate', 'int', 1000)
->setLocked(true)
->addExample(0, pht('No performance sampling.'))
->addExample(1, pht('Sample every request (slow).'))
->addExample(1000, pht('Sample 0.1%% of requests.'))
->setSummary(pht('Automatically sample some fraction of requests.'))
->setDescription(
pht(
"The Multimeter application collects performance samples. You ".
"can use this data to help you understand what Phabricator is ".
"spending time and resources doing, and to identify problematic ".
"access patterns.".
"\n\n".
"This option controls how frequently sampling activates. Set it ".
"to some positive integer N to sample every 1 / N pages.".
"\n\n".
"For most installs, the default value (1 sample per 1000 pages) ".
"should collect enough data to be useful without requiring much ".
"storage or meaningfully impacting performance. If you're ".
"investigating performance issues, you can adjust the rate ".
"in order to collect more data.")),
$this->newOption('phabricator.developer-mode', 'bool', false)
->setBoolOptions(
array(

View file

@ -0,0 +1,46 @@
<?php
final class PhabricatorMultimeterApplication
extends PhabricatorApplication {
public function getName() {
return pht('Multimeter');
}
public function getBaseURI() {
return '/multimeter/';
}
public function getFontIcon() {
return 'fa-motorcycle';
}
public function isPrototype() {
return true;
}
public function getTitleGlyph() {
return "\xE2\x8F\xB3";
}
public function getApplicationGroup() {
return self::GROUP_DEVELOPER;
}
public function getShortDescription() {
return pht('Performance Sampler');
}
public function getRemarkupRules() {
return array();
}
public function getRoutes() {
return array(
'/multimeter/' => array(
'' => 'MultimeterSampleController',
),
);
}
}

View file

@ -0,0 +1,70 @@
<?php
abstract class MultimeterController extends PhabricatorController {
private $dimensions = array();
protected function loadDimensions(array $rows) {
if (!$rows) {
return;
}
$map = array(
'eventLabelID' => new MultimeterLabel(),
'eventViewerID' => new MultimeterViewer(),
'eventHostID' => new MultimeterHost(),
'eventContextID' => new MultimeterContext(),
);
$ids = array();
foreach ($map as $key => $object) {
foreach ($rows as $row) {
$ids[$key][] = $row[$key];
}
}
foreach ($ids as $key => $list) {
$object = $map[$key];
if (empty($this->dimensions[$key])) {
$this->dimensions[$key] = array();
}
$this->dimensions[$key] += $object->loadAllWhere(
'id IN (%Ld)',
$list);
}
}
protected function getLabelDimension($id) {
if (empty($this->dimensions['eventLabelID'][$id])) {
return $this->newMissingDimension(new MultimeterLabel(), $id);
}
return $this->dimensions['eventLabelID'][$id];
}
protected function getViewerDimension($id) {
if (empty($this->dimensions['eventViewerID'][$id])) {
return $this->newMissingDimension(new MultimeterViewer(), $id);
}
return $this->dimensions['eventViewerID'][$id];
}
protected function getHostDimension($id) {
if (empty($this->dimensions['eventHostID'][$id])) {
return $this->newMissingDimension(new MultimeterHost(), $id);
}
return $this->dimensions['eventHostID'][$id];
}
protected function getContextDimension($id) {
if (empty($this->dimensions['eventContextID'][$id])) {
return $this->newMissingDimension(new MultimeterContext(), $id);
}
return $this->dimensions['eventContextID'][$id];
}
private function newMissingDimension(MultimeterDimension $dim, $id) {
$dim->setName('<missing:'.$id.'>');
return $dim;
}
}

View file

@ -0,0 +1,85 @@
<?php
final class MultimeterSampleController extends MultimeterController {
public function shouldAllowPublic() {
return true;
}
public function handleRequest(AphrontRequest $request) {
$viewer = $this->getViewer();
$table = new MultimeterEvent();
$conn = $table->establishConnection('r');
$data = queryfx_all(
$conn,
'SELECT * FROM %T ORDER BY id DESC LIMIT 100',
$table->getTableName());
$this->loadDimensions($data);
$rows = array();
foreach ($data as $row) {
$rows[] = array(
$row['id'],
$row['requestKey'],
$this->getViewerDimension($row['eventViewerID'])->getName(),
$this->getContextDimension($row['eventContextID'])->getName(),
$this->getHostDimension($row['eventHostID'])->getName(),
MultimeterEvent::getEventTypeName($row['eventType']),
$this->getLabelDimension($row['eventLabelID'])->getName(),
MultimeterEvent::formatResourceCost(
$viewer,
$row['eventType'],
$row['resourceCost']),
$row['sampleRate'],
phabricator_datetime($row['epoch'], $viewer),
);
}
$table = id(new AphrontTableView($rows))
->setHeaders(
array(
pht('ID'),
pht('Request'),
pht('Viewer'),
pht('Context'),
pht('Host'),
pht('Type'),
pht('Label'),
pht('Cost'),
pht('Rate'),
pht('Epoch'),
))
->setColumnClasses(
array(
null,
null,
null,
null,
null,
null,
'wide',
'n',
'n',
null,
));
$box = id(new PHUIObjectBoxView())
->setHeaderText(pht('Samples'))
->appendChild($table);
$crumbs = $this->buildApplicationCrumbs();
$crumbs->addTextCrumb(pht('Samples'));
return $this->buildApplicationPage(
array(
$crumbs,
$box,
),
array(
'title' => pht('Samples'),
));
}
}

View file

@ -0,0 +1,203 @@
<?php
final class MultimeterControl {
private static $instance;
private $events = array();
private $sampleRate;
private $pauseDepth;
private $eventViewer;
private $eventContext;
private function __construct() {
// Private.
}
public static function newInstance() {
$instance = new MultimeterControl();
// NOTE: We don't set the sample rate yet. This allows the multimeter to
// be initialized and begin recording events, then make a decision about
// whether the page will be sampled or not later on (once we've loaded
// enough configuration).
self::$instance = $instance;
return self::getInstance();
}
public static function getInstance() {
return self::$instance;
}
public function isActive() {
return ($this->sampleRate !== 0) && ($this->pauseDepth == 0);
}
public function setSampleRate($rate) {
if ($rate && (mt_rand(1, $rate) == $rate)) {
$sample_rate = $rate;
} else {
$sample_rate = 0;
}
$this->sampleRate = $sample_rate;
return;
}
public function pauseMultimeter() {
$this->pauseDepth++;
return $this;
}
public function unpauseMultimeter() {
if (!$this->pauseDepth) {
throw new Exception(pht('Trying to unpause an active multimeter!'));
}
$this->pauseDepth--;
return $this;
}
public function newEvent($type, $label, $cost) {
if (!$this->isActive()) {
return null;
}
$event = id(new MultimeterEvent())
->setEventType($type)
->setEventLabel($label)
->setResourceCost($cost)
->setEpoch(PhabricatorTime::getNow());
$this->events[] = $event;
return $event;
}
public function saveEvents() {
if (!$this->isActive()) {
return;
}
$events = $this->events;
if (!$events) {
return;
}
if ($this->sampleRate === null) {
throw new Exception(pht('Call setSampleRate() before saving events!'));
}
// Don't sample any of this stuff.
$this->pauseMultimeter();
$use_scope = AphrontWriteGuard::isGuardActive();
if ($use_scope) {
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
} else {
AphrontWriteGuard::allowDangerousUnguardedWrites(true);
}
$caught = null;
try {
$this->writeEvents();
} catch (Exception $ex) {
$caught = $ex;
}
if ($use_scope) {
unset($unguarded);
} else {
AphrontWriteGuard::allowDangerousUnguardedWrites(false);
}
$this->unpauseMultimeter();
if ($caught) {
throw $caught;
}
}
private function writeEvents() {
$events = $this->events;
$random = Filesystem::readRandomBytes(32);
$request_key = PhabricatorHash::digestForIndex($random);
$host_id = $this->loadHostID(php_uname('n'));
$context_id = $this->loadEventContextID($this->eventContext);
$viewer_id = $this->loadEventViewerID($this->eventViewer);
$label_map = $this->loadEventLabelIDs(mpull($events, 'getEventLabel'));
foreach ($events as $event) {
$event
->setRequestKey($request_key)
->setSampleRate($this->sampleRate)
->setEventHostID($host_id)
->setEventContextID($context_id)
->setEventViewerID($viewer_id)
->setEventLabelID($label_map[$event->getEventLabel()])
->save();
}
}
public function setEventContext($event_context) {
$this->eventContext = $event_context;
return $this;
}
public function setEventViewer($viewer) {
$this->eventViewer = $viewer;
return $this;
}
private function loadHostID($host) {
$map = $this->loadDimensionMap(new MultimeterHost(), array($host));
return idx($map, $host);
}
private function loadEventViewerID($viewer) {
$map = $this->loadDimensionMap(new MultimeterViewer(), array($viewer));
return idx($map, $viewer);
}
private function loadEventContextID($context) {
$map = $this->loadDimensionMap(new MultimeterContext(), array($context));
return idx($map, $context);
}
private function loadEventLabelIDs(array $labels) {
return $this->loadDimensionMap(new MultimeterLabel(), $labels);
}
private function loadDimensionMap(MultimeterDimension $table, array $names) {
$hashes = array();
foreach ($names as $name) {
$hashes[] = PhabricatorHash::digestForIndex($name);
}
$objects = $table->loadAllWhere('nameHash IN (%Ls)', $hashes);
$map = mpull($objects, 'getID', 'getName');
$need = array();
foreach ($names as $name) {
if (isset($map[$name])) {
continue;
}
$need[] = $name;
}
foreach ($need as $name) {
$object = id(clone $table)
->setName($name)
->save();
$map[$name] = $object->getID();
}
return $map;
}
}

View file

@ -0,0 +1,21 @@
<?php
final class MultimeterEventGarbageCollector
extends PhabricatorGarbageCollector {
public function collectGarbage() {
$ttl = phutil_units('90 days in seconds');
$table = new MultimeterEvent();
$conn_w = $table->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE epoch < %d LIMIT 100',
$table->getTableName(),
PhabricatorTime::getNow() - $ttl);
return ($conn_w->getAffectedRows() == 100);
}
}

View file

@ -0,0 +1,3 @@
<?php
final class MultimeterContext extends MultimeterDimension {}

View file

@ -0,0 +1,9 @@
<?php
abstract class MultimeterDAO extends PhabricatorLiskDAO {
public function getApplicationName() {
return 'multimeter';
}
}

View file

@ -0,0 +1,29 @@
<?php
abstract class MultimeterDimension extends MultimeterDAO {
protected $name;
protected $nameHash;
public function setName($name) {
$this->nameHash = PhabricatorHash::digestForIndex($name);
return parent::setName($name);
}
protected function getConfiguration() {
return array(
self::CONFIG_TIMESTAMPS => false,
self::CONFIG_COLUMN_SCHEMA => array(
'name' => 'text',
'nameHash' => 'bytes12',
),
self::CONFIG_KEY_SCHEMA => array(
'key_hash' => array(
'columns' => array('nameHash'),
'unique' => true,
),
),
) + parent::getConfiguration();
}
}

View file

@ -0,0 +1,71 @@
<?php
final class MultimeterEvent extends MultimeterDAO {
const TYPE_STATIC_RESOURCE = 0;
protected $eventType;
protected $eventLabelID;
protected $resourceCost;
protected $sampleRate;
protected $eventContextID;
protected $eventHostID;
protected $eventViewerID;
protected $epoch;
protected $requestKey;
private $eventLabel;
public function setEventLabel($event_label) {
$this->eventLabel = $event_label;
return $this;
}
public function getEventLabel() {
return $this->eventLabel;
}
public static function getEventTypeName($type) {
switch ($type) {
case self::TYPE_STATIC_RESOURCE:
return pht('Static Resource');
}
return pht('Unknown ("%s")', $type);
}
public static function formatResourceCost(
PhabricatorUser $viewer,
$type,
$cost) {
switch ($type) {
case self::TYPE_STATIC_RESOURCE:
return pht('%s Req', new PhutilNumber($cost));
}
return pht('%s Unit(s)', new PhutilNumber($cost));
}
protected function getConfiguration() {
return array(
self::CONFIG_TIMESTAMPS => false,
self::CONFIG_COLUMN_SCHEMA => array(
'eventType' => 'uint32',
'resourceCost' => 'sint64',
'sampleRate' => 'uint32',
'requestKey' => 'bytes12',
),
self::CONFIG_KEY_SCHEMA => array(
'key_request' => array(
'columns' => array('requestKey'),
),
'key_type' => array(
'columns' => array('eventType', 'epoch'),
),
),
) + parent::getConfiguration();
}
}

View file

@ -0,0 +1,3 @@
<?php
final class MultimeterHost extends MultimeterDimension {}

View file

@ -0,0 +1,3 @@
<?php
final class MultimeterLabel extends MultimeterDimension {}

View file

@ -0,0 +1,3 @@
<?php
final class MultimeterViewer extends MultimeterDimension {}

View file

@ -104,6 +104,7 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList {
'db.system' => array(),
'db.fund' => array(),
'db.almanac' => array(),
'db.multimeter' => array(),
'0000.legacy.sql' => array(
'legacy' => 0,
),