Add a robots.txt file to disallow /diffusion/

Summary:
Fixes T4610. Open to suggestions, etc., if there's anything I'm missing.

Also:

  - Moves these "system" endpoints into a real application.
  - Makes `isUnlisted()` work a little more consistently.

Test Plan: Accessed `/robots.txt`, `/status/` and `/debug/`.

Reviewers: chad, btrahan

Reviewed By: btrahan

Subscribers: aran, epriestley

Maniphest Tasks: T4610

Differential Revision: https://secure.phabricator.com/D8532

src/__phutil_library_map__.php

@@ -1129,6 +1129,7 @@ phutil_register_library_map(array(
     'PhabricatorApplicationSlowvote' => 'applications/slowvote/application/PhabricatorApplicationSlowvote.php',
     'PhabricatorApplicationStatusView' => 'applications/meta/view/PhabricatorApplicationStatusView.php',
     'PhabricatorApplicationSubscriptions' => 'applications/subscriptions/application/PhabricatorApplicationSubscriptions.php',
+    'PhabricatorApplicationSystem' => 'applications/system/application/PhabricatorApplicationSystem.php',
     'PhabricatorApplicationTest' => 'applications/base/controller/__tests__/PhabricatorApplicationTest.php',
     'PhabricatorApplicationTokens' => 'applications/tokens/application/PhabricatorApplicationTokens.php',
     'PhabricatorApplicationTransaction' => 'applications/transactions/storage/PhabricatorApplicationTransaction.php',
@@ -1420,7 +1421,7 @@ phutil_register_library_map(array(
     'PhabricatorDashboardTransactionQuery' => 'applications/dashboard/query/PhabricatorDashboardTransactionQuery.php',
     'PhabricatorDashboardViewController' => 'applications/dashboard/controller/PhabricatorDashboardViewController.php',
     'PhabricatorDataNotAttachedException' => 'infrastructure/storage/lisk/PhabricatorDataNotAttachedException.php',
-    'PhabricatorDebugController' => 'applications/system/PhabricatorDebugController.php',
+    'PhabricatorDebugController' => 'applications/system/controller/PhabricatorDebugController.php',
     'PhabricatorDefaultFileStorageEngineSelector' => 'applications/files/engineselector/PhabricatorDefaultFileStorageEngineSelector.php',
     'PhabricatorDefaultSearchEngineSelector' => 'applications/search/selector/PhabricatorDefaultSearchEngineSelector.php',
     'PhabricatorDeveloperConfigOptions' => 'applications/config/option/PhabricatorDeveloperConfigOptions.php',
@@ -1960,6 +1961,7 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryURINormalizerTestCase' => 'applications/repository/data/__tests__/PhabricatorRepositoryURINormalizerTestCase.php',
     'PhabricatorRepositoryURITestCase' => 'applications/repository/storage/__tests__/PhabricatorRepositoryURITestCase.php',
     'PhabricatorRepositoryVCSPassword' => 'applications/repository/storage/PhabricatorRepositoryVCSPassword.php',
+    'PhabricatorRobotsController' => 'applications/system/controller/PhabricatorRobotsController.php',
     'PhabricatorS3FileStorageEngine' => 'applications/files/engine/PhabricatorS3FileStorageEngine.php',
     'PhabricatorSQLPatchList' => 'infrastructure/storage/patch/PhabricatorSQLPatchList.php',
     'PhabricatorSSHKeyGenerator' => 'infrastructure/util/PhabricatorSSHKeyGenerator.php',
@@ -2077,7 +2079,7 @@ phutil_register_library_map(array(
     'PhabricatorStandardCustomFieldText' => 'infrastructure/customfield/standard/PhabricatorStandardCustomFieldText.php',
     'PhabricatorStandardCustomFieldUsers' => 'infrastructure/customfield/standard/PhabricatorStandardCustomFieldUsers.php',
     'PhabricatorStandardPageView' => 'view/page/PhabricatorStandardPageView.php',
-    'PhabricatorStatusController' => 'applications/system/PhabricatorStatusController.php',
+    'PhabricatorStatusController' => 'applications/system/controller/PhabricatorStatusController.php',
     'PhabricatorStorageFixtureScopeGuard' => 'infrastructure/testing/fixture/PhabricatorStorageFixtureScopeGuard.php',
     'PhabricatorStorageManagementAPI' => 'infrastructure/storage/management/PhabricatorStorageManagementAPI.php',
     'PhabricatorStorageManagementDatabasesWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDatabasesWorkflow.php',
@@ -3785,6 +3787,7 @@ phutil_register_library_map(array(
     'PhabricatorApplicationSlowvote' => 'PhabricatorApplication',
     'PhabricatorApplicationStatusView' => 'AphrontView',
     'PhabricatorApplicationSubscriptions' => 'PhabricatorApplication',
+    'PhabricatorApplicationSystem' => 'PhabricatorApplication',
     'PhabricatorApplicationTest' => 'PhabricatorApplication',
     'PhabricatorApplicationTokens' => 'PhabricatorApplication',
     'PhabricatorApplicationTransaction' =>
@@ -4759,6 +4762,7 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryURINormalizerTestCase' => 'PhabricatorTestCase',
     'PhabricatorRepositoryURITestCase' => 'PhabricatorTestCase',
     'PhabricatorRepositoryVCSPassword' => 'PhabricatorRepositoryDAO',
+    'PhabricatorRobotsController' => 'PhabricatorController',
     'PhabricatorS3FileStorageEngine' => 'PhabricatorFileStorageEngine',
     'PhabricatorSSHKeyGenerator' => 'Phobject',
     'PhabricatorSSHLog' => 'Phobject',

src/aphront/configuration/AphrontDefaultApplicationConfiguration.php

@@ -23,15 +23,9 @@ class AphrontDefaultApplicationConfiguration
         '' => 'DarkConsoleController',
         'data/(?P<key>[^/]+)/' => 'DarkConsoleDataController',
       ),
-
-      '/status/' => 'PhabricatorStatusController',
-
-
       '/help/' => array(
         'keyboardshortcut/' => 'PhabricatorHelpKeyboardShortcutController',
       ),
-
-      '/debug/' => 'PhabricatorDebugController',
     );
   }
 

src/applications/home/controller/PhabricatorHomeController.php

@@ -24,6 +24,7 @@ abstract class PhabricatorHomeController extends PhabricatorController {
     $applications = id(new PhabricatorApplicationQuery())
       ->setViewer($user)
       ->withInstalled(true)
+      ->withUnlisted(false)
       ->execute();
 
     foreach ($applications as $key => $application) {

src/applications/settings/panel/PhabricatorSettingsPanelHomePreferences.php

@@ -59,15 +59,10 @@ final class PhabricatorSettingsPanelHomePreferences
 
     $output = array();
 
-    $applications = PhabricatorApplication::getAllInstalledApplications();
+    $app_groups = mgroup($apps, 'getApplicationGroup');
+    $app_groups = array_select_keys($app_groups, array_keys($group_map));
 
-    $applications = mgroup($applications, 'getApplicationGroup');
-
-    $applications = array_select_keys(
-    $applications,
-    array_keys($group_map));
-
-    foreach ($applications as $group => $apps) {
+    foreach ($app_groups as $group => $apps) {
       $group_name = $group_map[$group];
       $rows = array();
 
@@ -81,8 +76,6 @@ final class PhabricatorSettingsPanelHomePreferences
           continue;
         }
 
-
-
         $default_name = PhabricatorApplication::getTileDisplayName($default);
 
         $hide = PhabricatorApplication::TILE_HIDE;

src/applications/system/application/PhabricatorApplicationSystem.php

@@ -0,0 +1,21 @@
+<?php
+
+final class PhabricatorApplicationSystem extends PhabricatorApplication {
+
+  public function canUninstall() {
+    return false;
+  }
+
+  public function isUnlisted() {
+    return true;
+  }
+
+  public function getRoutes() {
+    return array(
+      '/status/' => 'PhabricatorStatusController',
+      '/debug/' => 'PhabricatorDebugController',
+      '/robots.txt' => 'PhabricatorRobotsController',
+    );
+  }
+
+}

src/applications/system/controller/PhabricatorRobotsController.php

@@ -0,0 +1,37 @@
+<?php
+
+final class PhabricatorRobotsController extends PhabricatorController {
+
+  public function shouldRequireLogin() {
+    return false;
+  }
+
+  public function processRequest() {
+    $out = array();
+
+    // Prevent indexing of '/diffusion/', since the content is not generally
+    // useful to index, web spiders get stuck scraping the history of every
+    // file, and much of the content is Ajaxed in anyway so spiders won't even
+    // see it. These pages are also relatively expensive to generate.
+
+    // Note that this still allows commits (at '/rPxxxxx') to be indexed.
+    // They're probably not hugely useful, but suffer fewer of the problems
+    // Diffusion suffers and are hard to omit with 'robots.txt'.
+
+    $out[] = 'User-Agent: *';
+    $out[] = 'Disallow: /diffusion/';
+
+    // Add a small crawl delay (number of seconds between requests) for spiders
+    // which respect it. The intent here is to prevent spiders from affecting
+    // performance for users. The possible cost is slower indexing, but that
+    // seems like a reasonable tradeoff, since most Phabricator installs are
+    // probably not hugely concerned about cutting-edge SEO.
+    $out[] = 'Crawl-delay: 1';
+
+    $content = implode("\n", $out)."\n";
+
+    return id(new AphrontPlainTextResponse())
+      ->setContent($content)
+      ->setCacheDurationInSeconds(phutil_units('2 hours in seconds'));
+  }
+}