Improve routing of "/robots.txt", "/favicon.ico", and "/status/" on Short and Blog sites

Summary: Ref T13636. Add routing for "/robots.txt", "favicon.ico", and "/status/" on the ShortSite and BlogSite.

Test Plan: Visted all resources (and 404 pages) on Short and Blog sites, and Platform site.

Maniphest Tasks: T13636

Differential Revision: https://secure.phabricator.com/D21607

src/__phutil_library_map__.php

@@ -4689,9 +4689,11 @@ phutil_register_library_map(array(
     'PhabricatorRequestExceptionHandler' => 'aphront/handler/PhabricatorRequestExceptionHandler.php',
     'PhabricatorResetPasswordUserLogType' => 'applications/people/userlog/PhabricatorResetPasswordUserLogType.php',
     'PhabricatorResourceSite' => 'aphront/site/PhabricatorResourceSite.php',
-    'PhabricatorRobotsController' => 'applications/system/controller/PhabricatorRobotsController.php',
+    'PhabricatorRobotsBlogController' => 'applications/system/controller/robots/PhabricatorRobotsBlogController.php',
-    'PhabricatorRobotsPlatformController' => 'applications/system/controller/PhabricatorRobotsPlatformController.php',
+    'PhabricatorRobotsController' => 'applications/system/controller/robots/PhabricatorRobotsController.php',
-    'PhabricatorRobotsResourceController' => 'applications/system/controller/PhabricatorRobotsResourceController.php',
+    'PhabricatorRobotsPlatformController' => 'applications/system/controller/robots/PhabricatorRobotsPlatformController.php',
+    'PhabricatorRobotsResourceController' => 'applications/system/controller/robots/PhabricatorRobotsResourceController.php',
+    'PhabricatorRobotsShortController' => 'applications/system/controller/robots/PhabricatorRobotsShortController.php',
     'PhabricatorS3FileStorageEngine' => 'applications/files/engine/PhabricatorS3FileStorageEngine.php',
     'PhabricatorSMSAuthFactor' => 'applications/auth/factor/PhabricatorSMSAuthFactor.php',
     'PhabricatorSQLPatchList' => 'infrastructure/storage/patch/PhabricatorSQLPatchList.php',
@@ -11474,9 +11476,11 @@ phutil_register_library_map(array(
     'PhabricatorRequestExceptionHandler' => 'AphrontRequestExceptionHandler',
     'PhabricatorResetPasswordUserLogType' => 'PhabricatorUserLogType',
     'PhabricatorResourceSite' => 'PhabricatorSite',
+    'PhabricatorRobotsBlogController' => 'PhabricatorRobotsController',
     'PhabricatorRobotsController' => 'PhabricatorController',
     'PhabricatorRobotsPlatformController' => 'PhabricatorRobotsController',
     'PhabricatorRobotsResourceController' => 'PhabricatorRobotsController',
+    'PhabricatorRobotsShortController' => 'PhabricatorRobotsController',
     'PhabricatorS3FileStorageEngine' => 'PhabricatorFileStorageEngine',
     'PhabricatorSMSAuthFactor' => 'PhabricatorAuthFactor',
     'PhabricatorSQLPatchList' => 'Phobject',

src/applications/phame/application/PhabricatorPhameApplication.php

@@ -70,7 +70,11 @@ final class PhabricatorPhameApplication extends PhabricatorApplication {
   }
 
   public function getBlogRoutes() {
-    return $this->getLiveRoutes();
+    return $this->getLiveRoutes() + array(
+      '/status/' => 'PhabricatorStatusController',
+      '/favicon.ico' => 'PhabricatorFaviconController',
+      '/robots.txt' => 'PhabricatorRobotsBlogController',
+    );
   }
 
   private function getLiveRoutes() {

src/applications/phurl/application/PhabricatorPhurlApplication.php

@@ -55,6 +55,10 @@ final class PhabricatorPhurlApplication extends PhabricatorApplication {
 
   public function getShortRoutes() {
     return array(
+      '/status/' => 'PhabricatorStatusController',
+      '/favicon.ico' => 'PhabricatorFaviconController',
+      '/robots.txt' => 'PhabricatorRobotsShortController',
+
       '/u/(?P<append>[^/]+)' => 'PhabricatorPhurlShortURLController',
       '.*' => 'PhabricatorPhurlShortURLDefaultController',
     );

src/applications/system/controller/robots/PhabricatorRobotsBlogController.php

@@ -0,0 +1,17 @@
+<?php
+
+final class PhabricatorRobotsBlogController
+  extends PhabricatorRobotsController {
+
+  protected function newRobotsRules() {
+    $out = array();
+
+    // Allow everything on blog domains to be indexed.
+
+    $out[] = 'User-Agent: *';
+    $out[] = 'Crawl-delay: 1';
+
+    return $out;
+  }
+
+}

src/applications/system/controller/robots/PhabricatorRobotsController.php

@@ -9,13 +9,6 @@ abstract class PhabricatorRobotsController extends PhabricatorController {
   final public function processRequest() {
     $out = $this->newRobotsRules();
 
-    // Add a small crawl delay (number of seconds between requests) for spiders
-    // which respect it. The intent here is to prevent spiders from affecting
-    // performance for users. The possible cost is slower indexing, but that
-    // seems like a reasonable tradeoff, since most Phabricator installs are
-    // probably not hugely concerned about cutting-edge SEO.
-    $out[] = 'Crawl-delay: 1';
-
     $content = implode("\n", $out)."\n";
 
     return id(new AphrontPlainTextResponse())

src/applications/system/controller/robots/PhabricatorRobotsPlatformController.php

@@ -19,6 +19,13 @@ final class PhabricatorRobotsPlatformController
     $out[] = 'Disallow: /diffusion/';
     $out[] = 'Disallow: /source/';
 
+    // Add a small crawl delay (number of seconds between requests) for spiders
+    // which respect it. The intent here is to prevent spiders from affecting
+    // performance for users. The possible cost is slower indexing, but that
+    // seems like a reasonable tradeoff, since most Phabricator installs are
+    // probably not hugely concerned about cutting-edge SEO.
+    $out[] = 'Crawl-delay: 1';
+
     return $out;
   }
 

src/applications/system/controller/robots/PhabricatorRobotsResourceController.php

@@ -10,6 +10,7 @@ final class PhabricatorRobotsResourceController
 
     $out[] = 'User-Agent: *';
     $out[] = 'Disallow: /';
+    $out[] = 'Crawl-delay: 1';
 
     return $out;
   }

src/applications/system/controller/robots/PhabricatorRobotsShortController.php

@@ -0,0 +1,18 @@
+<?php
+
+final class PhabricatorRobotsShortController
+  extends PhabricatorRobotsController {
+
+  protected function newRobotsRules() {
+    $out = array();
+
+    // See T13636. Prevent indexing of any content on short domains.
+
+    $out[] = 'User-Agent: *';
+    $out[] = 'Disallow: /';
+    $out[] = 'Crawl-delay: 1';
+
+    return $out;
+  }
+
+}