Improve routing of "/robots.txt", "/favicon.ico", "/status/", and 404 on custom Sites

Summary:
Fixes T12919. Fixes T13636. Prior to this change, some well-known resource paths don't route on sites like ResourceSite.

- `/robots.txt`: Make it route on ResourceSite and just deny the whole site.
- `/favicon.ico`: Make it route on ResourceSite.
- `/status/`: Make it route on ResourceSite.
- 404: Make it render a 404 on ResourceSite.

Test Plan:
  - Visited all URIs on ResourceSite, got sensible responses.
  - Visited all URIs on main site.
  - Visited 404 while logged out, got login page.

Maniphest Tasks: T13636, T12919

Differential Revision: https://secure.phabricator.com/D21606

src/__phutil_library_map__.php

@@ -4253,6 +4253,7 @@ phutil_register_library_map(array(
     'PhabricatorPhurlURLViewController' => 'applications/phurl/controller/PhabricatorPhurlURLViewController.php',
     'PhabricatorPinnedApplicationsSetting' => 'applications/settings/setting/PhabricatorPinnedApplicationsSetting.php',
     'PhabricatorPirateEnglishTranslation' => 'infrastructure/internationalization/translation/PhabricatorPirateEnglishTranslation.php',
+    'PhabricatorPlatform404Controller' => 'applications/base/controller/PhabricatorPlatform404Controller.php',
     'PhabricatorPlatformSite' => 'aphront/site/PhabricatorPlatformSite.php',
     'PhabricatorPointsEditField' => 'applications/transactions/editfield/PhabricatorPointsEditField.php',
     'PhabricatorPointsFact' => 'applications/fact/fact/PhabricatorPointsFact.php',
@@ -4689,6 +4690,8 @@ phutil_register_library_map(array(
     'PhabricatorResetPasswordUserLogType' => 'applications/people/userlog/PhabricatorResetPasswordUserLogType.php',
     'PhabricatorResourceSite' => 'aphront/site/PhabricatorResourceSite.php',
     'PhabricatorRobotsController' => 'applications/system/controller/PhabricatorRobotsController.php',
+    'PhabricatorRobotsPlatformController' => 'applications/system/controller/PhabricatorRobotsPlatformController.php',
+    'PhabricatorRobotsResourceController' => 'applications/system/controller/PhabricatorRobotsResourceController.php',
     'PhabricatorS3FileStorageEngine' => 'applications/files/engine/PhabricatorS3FileStorageEngine.php',
     'PhabricatorSMSAuthFactor' => 'applications/auth/factor/PhabricatorSMSAuthFactor.php',
     'PhabricatorSQLPatchList' => 'infrastructure/storage/patch/PhabricatorSQLPatchList.php',
@@ -10914,6 +10917,7 @@ phutil_register_library_map(array(
     'PhabricatorPhurlURLViewController' => 'PhabricatorPhurlController',
     'PhabricatorPinnedApplicationsSetting' => 'PhabricatorInternalSetting',
     'PhabricatorPirateEnglishTranslation' => 'PhutilTranslation',
+    'PhabricatorPlatform404Controller' => 'PhabricatorController',
     'PhabricatorPlatformSite' => 'PhabricatorSite',
     'PhabricatorPointsEditField' => 'PhabricatorEditField',
     'PhabricatorPointsFact' => 'PhabricatorFact',
@@ -11471,6 +11475,8 @@ phutil_register_library_map(array(
     'PhabricatorResetPasswordUserLogType' => 'PhabricatorUserLogType',
     'PhabricatorResourceSite' => 'PhabricatorSite',
     'PhabricatorRobotsController' => 'PhabricatorController',
+    'PhabricatorRobotsPlatformController' => 'PhabricatorRobotsController',
+    'PhabricatorRobotsResourceController' => 'PhabricatorRobotsController',
     'PhabricatorS3FileStorageEngine' => 'PhabricatorFileStorageEngine',
     'PhabricatorSMSAuthFactor' => 'PhabricatorAuthFactor',
     'PhabricatorSQLPatchList' => 'Phobject',

src/aphront/configuration/AphrontApplicationConfiguration.php

@@ -32,10 +32,6 @@ final class AphrontApplicationConfiguration
     return $request;
   }
 
-  public function build404Controller() {
-    return array(new Phabricator404Controller(), array());
-  }
-
   public function buildRedirectController($uri, $external) {
     return array(
       new PhabricatorRedirectController(),
@@ -504,7 +500,10 @@ final class AphrontApplicationConfiguration
       return array($result, array());
     }
 
-    return $this->build404Controller();
+    throw new Exception(
+      pht(
+        'Aphront site ("%s") failed to build a 404 controller.',
+        get_class($site)));
   }
 
   /**

src/aphront/response/Aphront404Response.php

@@ -10,10 +10,17 @@ final class Aphront404Response extends AphrontHTMLResponse {
     $request = $this->getRequest();
     $viewer = $request->getViewer();
 
+    // See T13636. Note that this response may be served from a Site other than
+    // the primary PlatformSite. For now, always link to the PlatformSite.
+
+    // (This may not be the best possible place to send users who are currently
+    // on "real" sites, like the BlogSite.)
+    $return_uri = PhabricatorEnv::getURI('/');
+
     $dialog = id(new AphrontDialogView())
       ->setViewer($viewer)
       ->setTitle(pht('404 Not Found'))
-      ->addCancelButton('/', pht('Return to Charted Waters'))
+      ->addCancelButton($return_uri, pht('Return to Charted Waters'))
       ->appendParagraph(
         pht(
           'You arrive at your destination, but there is nothing here.'))

src/aphront/site/AphrontSite.php

@@ -10,7 +10,7 @@ abstract class AphrontSite extends Phobject {
   abstract public function getRoutingMaps();
 
   public function new404Controller(AphrontRequest $request) {
-    return null;
+    return new Phabricator404Controller();
   }
 
   protected function isHostMatch($host, array $uris) {

src/aphront/site/PhabricatorPlatformSite.php

@@ -50,4 +50,8 @@ final class PhabricatorPlatformSite extends PhabricatorSite {
     return $maps;
   }
 
+  public function new404Controller(AphrontRequest $request) {
+    return new PhabricatorPlatform404Controller();
+  }
+
 }

src/applications/auth/extension/PhabricatorAuthMainMenuBarExtension.php

@@ -39,7 +39,13 @@ final class PhabricatorAuthMainMenuBarExtension
   private function buildLoginMenu() {
     $controller = $this->getController();
 
-    $uri = new PhutilURI('/auth/start/');
+    // See T13636. This button may be rendered by the 404 controller on sites
+    // other than the primary PlatformSite. Link the button to the primary
+    // site.
+
+    $uri = '/auth/start/';
+    $uri = PhabricatorEnv::getURI($uri);
+    $uri = new PhutilURI($uri);
     if ($controller) {
       $path = $controller->getRequest()->getPath();
       $uri->replaceQueryParam('next', $path);

src/applications/base/controller/Phabricator404Controller.php

@@ -1,6 +1,11 @@
 <?php
 
-final class Phabricator404Controller extends PhabricatorController {
+final class Phabricator404Controller
+  extends PhabricatorController {
+
+  public function shouldRequireLogin() {
+    return false;
+  }
 
   public function processRequest() {
     return new Aphront404Response();

src/applications/base/controller/PhabricatorPlatform404Controller.php

@@ -0,0 +1,10 @@
+<?php
+
+final class PhabricatorPlatform404Controller
+  extends PhabricatorController {
+
+  public function processRequest() {
+    return new Aphront404Response();
+  }
+
+}

src/applications/system/application/PhabricatorSystemApplication.php

@@ -25,7 +25,7 @@ final class PhabricatorSystemApplication extends PhabricatorApplication {
       '/status/' => 'PhabricatorStatusController',
       '/debug/' => 'PhabricatorDebugController',
       '/favicon.ico' => 'PhabricatorFaviconController',
-      '/robots.txt' => 'PhabricatorRobotsController',
+      '/robots.txt' => 'PhabricatorRobotsPlatformController',
       '/services/' => array(
         'encoding/' => 'PhabricatorSystemSelectEncodingController',
         'highlight/' => 'PhabricatorSystemSelectHighlightController',
@@ -38,4 +38,12 @@ final class PhabricatorSystemApplication extends PhabricatorApplication {
     );
   }
 
+  public function getResourceRoutes() {
+    return array(
+      '/status/' => 'PhabricatorStatusController',
+      '/favicon.ico' => 'PhabricatorFaviconController',
+      '/robots.txt' => 'PhabricatorRobotsResourceController',
+    );
+  }
+
 }

src/applications/system/controller/PhabricatorRobotsController.php

@@ -1,26 +1,13 @@
 <?php
 
-final class PhabricatorRobotsController extends PhabricatorController {
+abstract class PhabricatorRobotsController extends PhabricatorController {
 
   public function shouldRequireLogin() {
     return false;
   }
 
-  public function processRequest() {
-    $out = array();
-
-    // Prevent indexing of '/diffusion/', since the content is not generally
-    // useful to index, web spiders get stuck scraping the history of every
-    // file, and much of the content is Ajaxed in anyway so spiders won't even
-    // see it. These pages are also relatively expensive to generate.
-
-    // Note that this still allows commits (at '/rPxxxxx') to be indexed.
-    // They're probably not hugely useful, but suffer fewer of the problems
-    // Diffusion suffers and are hard to omit with 'robots.txt'.
-
-    $out[] = 'User-Agent: *';
-    $out[] = 'Disallow: /diffusion/';
-    $out[] = 'Disallow: /source/';
+  final public function processRequest() {
+    $out = $this->newRobotsRules();
 
     // Add a small crawl delay (number of seconds between requests) for spiders
     // which respect it. The intent here is to prevent spiders from affecting
@@ -36,4 +23,7 @@ final class PhabricatorRobotsController extends PhabricatorController {
       ->setCacheDurationInSeconds(phutil_units('2 hours in seconds'))
       ->setCanCDN(true);
   }
+
+  abstract protected function newRobotsRules();
+
 }

src/applications/system/controller/PhabricatorRobotsPlatformController.php

@@ -0,0 +1,25 @@
+<?php
+
+final class PhabricatorRobotsPlatformController
+  extends PhabricatorRobotsController {
+
+  protected function newRobotsRules() {
+    $out = array();
+
+    // Prevent indexing of '/diffusion/', since the content is not generally
+    // useful to index, web spiders get stuck scraping the history of every
+    // file, and much of the content is Ajaxed in anyway so spiders won't even
+    // see it. These pages are also relatively expensive to generate.
+
+    // Note that this still allows commits (at '/rPxxxxx') to be indexed.
+    // They're probably not hugely useful, but suffer fewer of the problems
+    // Diffusion suffers and are hard to omit with 'robots.txt'.
+
+    $out[] = 'User-Agent: *';
+    $out[] = 'Disallow: /diffusion/';
+    $out[] = 'Disallow: /source/';
+
+    return $out;
+  }
+
+}

src/applications/system/controller/PhabricatorRobotsResourceController.php

@@ -0,0 +1,17 @@
+<?php
+
+final class PhabricatorRobotsResourceController
+  extends PhabricatorRobotsController {
+
+  protected function newRobotsRules() {
+    $out = array();
+
+    // See T13636. Prevent indexing of any content on resource domains.
+
+    $out[] = 'User-Agent: *';
+    $out[] = 'Disallow: /';
+
+    return $out;
+  }
+
+}