Improve DarkConsole "Services" and "XHProf" plugins

Summary:

  - Services: Show summary panel of total service call costs and relative page weight.
  - Services: Add "Analyze Query Plans" button, which issues EXPLAIN for each query and flags problems.
  - XHPRof: iframe the profile.

Test Plan: Used the new query plan analysis to find missing keys causing table scans, see D627.

Reviewers: jungejason, tuomaspelkonen, aran

CC:

Differential Revision: 628

src/__celerity_resource_map__.php

@@ -27,7 +27,7 @@ celerity_register_resource_map(array(
   ),
   'aphront-dark-console-css' =>
   array(
-    'uri' => '/res/e7011594/rsrc/css/aphront/dark-console.css',
+    'uri' => '/res/a7d1dbf1/rsrc/css/aphront/dark-console.css',
     'type' => 'css',
     'requires' =>
     array(

src/aphront/console/plugin/base/DarkConsolePlugin.php

@@ -61,6 +61,10 @@ abstract class DarkConsolePlugin {
     return $this->request;
   }
 
+  public function getRequestURI() {
+    return $this->getRequest()->getRequestURI();
+  }
+
   public function isPermanent() {
     return false;
   }

src/aphront/console/plugin/services/DarkConsoleServicesPlugin.php

@@ -30,22 +30,198 @@ class DarkConsoleServicesPlugin extends DarkConsolePlugin {
 
   public function generateData() {
 
+    $log = PhutilServiceProfiler::getInstance()->getServiceCallLog();
+    foreach ($log as $key => $entry) {
+      $config = $entry['config'];
+      unset($log[$key]['config']);
+
+      if (empty($_REQUEST['__analyze__'])) {
+        $log[$key]['explain'] = array(
+          'sev'     => 7,
+          'size'    => null,
+          'reason'  => 'Disabled',
+        );
+        // Query analysis is disabled for this request, so don't do any of it.
+        continue;
+      }
+
+      if ($entry['type'] != 'query') {
+        continue;
+      }
+
+      // For each SELECT query, go issue an EXPLAIN on it so we can flag stuff
+      // causing table scans, etc.
+      if (preg_match('/^\s*SELECT\b/i', $entry['query'])) {
+        $conn = new AphrontMySQLDatabaseConnection($entry['config']);
+        try {
+          $explain = queryfx_all(
+            $conn,
+            'EXPLAIN %Q',
+            $entry['query']);
+
+          $badness = 0;
+          $size    = 1;
+          $reason  = null;
+
+          foreach ($explain as $table) {
+            $size *= (int)$table['rows'];
+
+            switch ($table['type']) {
+              case 'index':
+                $cur_badness = 1;
+                $cur_reason  = 'Index';
+                break;
+              case 'const':
+                $cur_badness = 1;
+                $cur_reason = 'Const';
+                break;
+              case 'eq_ref';
+                $cur_badness = 2;
+                $cur_reason = 'EqRef';
+                break;
+              case 'range':
+                $cur_badness = 3;
+                $cur_reason = 'Range';
+                break;
+              case 'ref':
+                $cur_badness = 3;
+                $cur_reason = 'Ref';
+                break;
+              case 'ALL':
+                if (preg_match('/Using where/', $table['Extra'])) {
+                  if ($table['rows'] < 256 && !empty($table['possible_keys'])) {
+                    $cur_badness = 2;
+                    $cur_reason = 'Small Table Scan';
+                  } else {
+                    $cur_badness = 6;
+                    $cur_reason = 'TABLE SCAN!';
+                  }
+                } else {
+                  $cur_badness = 3;
+                  $cur_reason = 'Whole Table';
+                }
+                break;
+              default:
+                if (preg_match('/No tables used/i', $table['Extra'])) {
+                  $cur_badness = 1;
+                  $cur_reason = 'No Tables';
+                } else if (preg_match('/Impossible/i', $table['Extra'])) {
+                  $cur_badness = 1;
+                  $cur_reason = 'Empty';
+                } else {
+                  $cur_badness = 4;
+                  $cur_reason = "Can't Analyze";
+                }
+                break;
+            }
+
+            if ($cur_badness > $badness) {
+              $badness = $cur_badness;
+              $reason = $cur_reason;
+            }
+          }
+
+          $log[$key]['explain'] = array(
+            'sev'     => $badness,
+            'size'    => $size,
+            'reason'  => $reason,
+          );
+        } catch (Exception $ex) {
+          $log[$key]['explain'] = array(
+            'sev'     => 5,
+            'size'    => null,
+            'reason'  => $ex->getMessage(),
+          );
+        }
+      }
+    }
+
     return array(
       'start' => $GLOBALS['__start__'],
-      'log'   => PhutilServiceProfiler::getInstance()->getServiceCallLog(),
+      'end'   => microtime(true),
+      'log'   => $log,
     );
   }
 
   public function render() {
     $data = $this->getData();
     $log = $data['log'];
+    $results = array();
+
+    $results[] =
+      '<div class="dark-console-panel-header">'.
+        phutil_render_tag(
+          'a',
+          array(
+            'href'  => $this->getRequestURI()->alter('__analyze__', true),
+            'class' => isset($_REQUEST['__analyze__'])
+              ? 'disabled button'
+              : 'green button',
+          ),
+          'Analyze Query Plans').
+        '<h1>Calls to External Services</h1>'.
+        '<div style="clear: both;"></div>'.
+      '</div>';
+
+    $page_total = $data['end'] - $data['start'];
+    $totals = array();
+    $counts = array();
+
+    foreach ($log as $row) {
+      $totals[$row['type']] += $row['duration'];
+      $counts[$row['type']]++;
+    }
+    $totals['All'] = array_sum($totals);
+    $counts['All'] = array_sum($counts);
+
+    $table = new AphrontTableView();
+    $summary = array();
+    foreach ($totals as $type => $total) {
+      $summary[] = array(
+        $type,
+        number_format($counts[$type]),
+        number_format((int)(1000000 * $totals[$type])).' us',
+        sprintf('%.1f%%', 100 * $totals[$type] / $page_total),
+      );
+    }
+    $summary_table = new AphrontTableView($summary);
+    $summary_table->setColumnClasses(
+      array(
+        '',
+        'n',
+        'n',
+        'wide',
+      ));
+    $summary_table->setHeaders(
+      array(
+        'Type',
+        'Count',
+        'Total Cost',
+        'Page Weight',
+      ));
+
+    $results[] = $summary_table->render();
 
     $rows = array();
     foreach ($log as $row) {
 
+      $analysis = null;
+
       switch ($row['type']) {
         case 'query':
           $info = $row['query'];
+          $info = wordwrap($info, 128, "\n", true);
+
+          if (!empty($row['explain'])) {
+            $analysis = phutil_escape_html($row['explain']['reason']);
+            $analysis = phutil_render_tag(
+              'span',
+              array(
+                'class' => 'explain-sev-'.$row['explain']['sev'],
+              ),
+              $analysis);
+          }
+
           $info = phutil_escape_html($info);
           break;
         case 'connect':
@@ -70,6 +246,7 @@ class DarkConsoleServicesPlugin extends DarkConsolePlugin {
         '+'.number_format(1000 * ($row['begin'] - $data['start'])).' ms',
         number_format(1000000 * $row['duration']).' us',
         $info,
+        $analysis,
       );
     }
 
@@ -79,7 +256,8 @@ class DarkConsoleServicesPlugin extends DarkConsolePlugin {
         null,
         'n',
         'n',
-        'wide wrap',
+        'wide',
+        '',
       ));
     $table->setHeaders(
       array(
@@ -87,9 +265,12 @@ class DarkConsoleServicesPlugin extends DarkConsolePlugin {
         'Start',
         'Duration',
         'Details',
+        'Analysis',
       ));
 
-    return $table->render();
+    $results[] = $table->render();
+
+    return implode("\n", $results);
   }
 }
 

src/aphront/console/plugin/services/__init__.php

@@ -7,6 +7,8 @@
 
 
 phutil_require_module('phabricator', 'aphront/console/plugin/base');
+phutil_require_module('phabricator', 'storage/connection/mysql');
+phutil_require_module('phabricator', 'storage/queryfx');
 phutil_require_module('phabricator', 'view/control/table');
 
 phutil_require_module('phutil', 'markup');

src/aphront/console/plugin/xhprof/DarkConsoleXHProfPlugin.php

@@ -44,36 +44,58 @@ class DarkConsoleXHProfPlugin extends DarkConsolePlugin {
 
   public function render() {
     if (!DarkConsoleXHProfPluginAPI::isProfilerAvailable()) {
+      $href = PhabricatorEnv::getDoclink('article/Installation_Guide.html');
+      $install_guide = phutil_render_tag(
+        'a',
+        array(
+          'href' => $href,
+          'class' => 'bright-link',
+        ),
+        'Installation Guide');
       return
-        '<p>The "xhprof" PHP extension is not available. Install xhprof '.
+        '<div class="dark-console-no-content">'.
-        'to enable the XHProf plugin.';
+          'The "xhprof" PHP extension is not available. Install xhprof '.
+          'to enable the XHProf console plugin. You can find instructions in '.
+          'the '.$install_guide.'.'.
+        '</div>';
     }
 
+    $result = array();
+
     $run = $this->getXHProfRunID();
-    if ($run) {
-      return '<a href="/xhprof/profile/'.$run.'/">View Run</a>';
-    } else {
-      $hidden = array();
-      $data = array('__profile__' => 'page') + $_GET;
 
-      foreach ($data as $k => $v) {
+    $header =
-        $hidden[] = phutil_render_tag(
+      '<div class="dark-console-panel-header">'.
-          'input',
+        phutil_render_tag(
+          'a',
           array(
-            'type' => 'hidden',
+            'href'  => $this->getRequestURI()->alter('__profile__', 'page'),
-            'name' => $k,
+            'class' => $run
-            'value' => $v,
+              ? 'disabled button'
-          ));
+              : 'green button',
-      }
+          ),
-      $hidden = implode("\n", $hidden);
+          'Profile Page').
+        '<h1>XHProf Profiler</h1>'.
+      '</div>';
+    $result[] = $header;
 
-
+    if ($run) {
-      return
+      $result[] =
-        '<form method="get">'.
+        '<a href="/xhprof/profile/'.$run.'/" '.
-          $hidden.
+          'class="bright-link" '.
-          '<button>Enable XHProf</button>'.
+          'style="float: right; margin: 1em 2em 0 0;'.
-        '</form>';
+            'font-weight: bold;" '.
+          'target="_blank">Profile Permalink</a>'.
+        '<iframe src="/xhprof/profile/'.$run.'/?frame=true"></iframe>';
+    } else {
+      $result[] =
+        '<div class="dark-console-no-content">'.
+          'Profiling was not enabled for this page. Use the button above '.
+          'to enable it.'.
+        '</div>';
     }
+
+    return implode("\n", $result);
   }
 
 

src/aphront/console/plugin/xhprof/__init__.php

@@ -8,6 +8,7 @@
 
 phutil_require_module('phabricator', 'aphront/console/plugin/base');
 phutil_require_module('phabricator', 'aphront/console/plugin/xhprof/api');
+phutil_require_module('phabricator', 'infrastructure/env');
 
 phutil_require_module('phutil', 'markup');
 

src/applications/files/controller/list/PhabricatorFileListController.php

@@ -22,6 +22,7 @@ class PhabricatorFileListController extends PhabricatorFileController {
 
     $request = $this->getRequest();
 
+    $author = null;
     $author_username = $request->getStr('author');
     if ($author_username) {
       $author = id(new PhabricatorUser())->loadOneWhere(

src/applications/xhprof/controller/base/PhabricatorXHProfController.php

@@ -28,6 +28,14 @@ abstract class PhabricatorXHProfController extends PhabricatorController {
     $page->appendChild($view);
 
     $response = new AphrontWebpageResponse();
+
+    if (isset($data['frame'])) {
+      $response->setFrameable(true);
+      $page->setFrameable(true);
+      $page->setShowChrome(false);
+      $page->setDisableConsole(true);
+    }
+
     return $response->setContent($page->render());
   }
 

src/applications/xhprof/controller/profile/PhabricatorXHProfProfileController.php

@@ -58,6 +58,7 @@ class PhabricatorXHProfProfileController
       $view,
       array(
         'title' => 'Profile',
+        'frame' => $request->getBool('frame'),
       ));
   }
 }

src/storage/connection/mysql/AphrontMySQLDatabaseConnection.php

@@ -213,8 +213,9 @@ class AphrontMySQLDatabaseConnection extends AphrontDatabaseConnection {
         $profiler = PhutilServiceProfiler::getInstance();
         $call_id = $profiler->beginServiceCall(
           array(
-            'type'  => 'query',
+            'type'    => 'query',
-            'query' => $raw_query,
+            'config'  => $this->configuration,
+            'query'   => $raw_query,
           ));
 
         $result = @mysql_query($raw_query, $this->connection);

src/view/page/standard/PhabricatorStandardPageView.php

@@ -27,6 +27,8 @@ class PhabricatorStandardPageView extends AphrontPageView {
   private $request;
   private $isAdminInterface;
   private $showChrome = true;
+  private $isFrameable = false;
+  private $disableConsole;
 
   public function setIsAdminInterface($is_admin_interface) {
     $this->isAdminInterface = $is_admin_interface;
@@ -51,6 +53,16 @@ class PhabricatorStandardPageView extends AphrontPageView {
     return $this;
   }
 
+  public function setFrameable($frameable) {
+    $this->isFrameable = $frameable;
+    return $this;
+  }
+
+  public function setDisableConsole($disable) {
+    $this->disableConsole = $disable;
+    return $this;
+  }
+
   public function getApplicationName() {
     return $this->applicationName;
   }
@@ -103,7 +115,7 @@ class PhabricatorStandardPageView extends AphrontPageView {
         "You must set the Request to render a PhabricatorStandardPageView.");
     }
 
-    $console = $this->getRequest()->getApplicationConfiguration()->getConsole();
+    $console = $this->getConsole();
 
     require_celerity_resource('phabricator-core-css');
     require_celerity_resource('phabricator-core-buttons-css');
@@ -133,10 +145,16 @@ class PhabricatorStandardPageView extends AphrontPageView {
 
 
   protected function getHead() {
+
+    $framebust = null;
+    if (!$this->isFrameable) {
+      $framebust = '(top != self) && top.location.replace(self.location.href);';
+    }
+
     $response = CelerityAPI::getStaticResourceResponse();
     $head =
       '<script type="text/javascript">'.
-        '(top != self) && top.location.replace(self.location.href);'.
+        $framebust.
         'window.__DEV__=1;'.
       '</script>'.
       $response->renderResourcesOfType('css').
@@ -185,7 +203,7 @@ class PhabricatorStandardPageView extends AphrontPageView {
   }
 
   protected function getBody() {
-    $console = $this->getRequest()->getApplicationConfiguration()->getConsole();
+    $console = $this->getConsole();
 
     $tabs = array();
     foreach ($this->tabs as $name => $tab) {
@@ -345,4 +363,10 @@ class PhabricatorStandardPageView extends AphrontPageView {
     return implode(' ', $classes);
   }
 
+  private function getConsole() {
+    if ($this->disableConsole) {
+      return null;
+    }
+    return $this->getRequest()->getApplicationConfiguration()->getConsole();
+  }
 }

webroot/rsrc/css/aphront/dark-console.css

@@ -87,3 +87,68 @@ a.dark-console-tab-selected {
     height: 2px;
 }
 
+.explain-sev-1 {
+  color: #33ff33;
+}
+
+.explain-sev-2 {
+  color: #99ff33;
+}
+
+.explain-sev-3 {
+  color: #ccff33;
+}
+
+.explain-sev-4 {
+  color: #ffff33;
+}
+
+.explain-sev-5 {
+  color: #ffcc33;
+}
+
+.explain-sev-6 {
+  color: #ffffff;
+  font-weight: bold;
+  background: #aa0000;
+  padding: 0 1em;
+  border: 2px solid #ffff00;
+}
+
+.explain-sev-7 {
+  color: #aaaaaa;
+}
+
+.dark-console-panel-header {
+  background: #606060;
+  border-bottom: 1px solid #505050;
+  padding: .25em 1em .25em 0;
+}
+
+.dark-console-panel-header h1 {
+  padding: 1em;
+  font-size: 12px;
+  font-weight: normal;
+}
+
+.dark-console-panel-header .button {
+  margin-top: .5em;
+  float: right;
+}
+
+.dark-console-panel a.bright-link {
+  color: #00cfff;
+  font-weight: bold;
+}
+
+.dark-console iframe {
+  width: 98%;
+  margin: .5em 1%;
+  height: 450px;
+  border: 0;
+}
+
+.dark-console-no-content {
+  padding: 1.5em 2em;
+  font-style: italic;
+}