From 31c9d4094fa54a32bb04a307d39b6b26c8727826 Mon Sep 17 00:00:00 2001 From: epriestley Date: Thu, 11 Mar 2021 13:47:23 -0800 Subject: [PATCH] Improve routing of "/robots.txt", "/favicon.ico", and "/status/" on Short and Blog sites Summary: Ref T13636. Add routing for "/robots.txt", "favicon.ico", and "/status/" on the ShortSite and BlogSite. Test Plan: Visted all resources (and 404 pages) on Short and Blog sites, and Platform site. Maniphest Tasks: T13636 Differential Revision: https://secure.phabricator.com/D21607 --- src/__phutil_library_map__.php | 10 +++++++--- .../PhabricatorPhameApplication.php | 6 +++++- .../PhabricatorPhurlApplication.php | 4 ++++ .../robots/PhabricatorRobotsBlogController.php | 17 +++++++++++++++++ .../PhabricatorRobotsController.php | 7 ------- .../PhabricatorRobotsPlatformController.php | 7 +++++++ .../PhabricatorRobotsResourceController.php | 1 + .../PhabricatorRobotsShortController.php | 18 ++++++++++++++++++ 8 files changed, 59 insertions(+), 11 deletions(-) create mode 100644 src/applications/system/controller/robots/PhabricatorRobotsBlogController.php rename src/applications/system/controller/{ => robots}/PhabricatorRobotsController.php (55%) rename src/applications/system/controller/{ => robots}/PhabricatorRobotsPlatformController.php (66%) rename src/applications/system/controller/{ => robots}/PhabricatorRobotsResourceController.php (90%) create mode 100644 src/applications/system/controller/robots/PhabricatorRobotsShortController.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 9f02ad2c41..3cae40d75e 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -4689,9 +4689,11 @@ phutil_register_library_map(array( 'PhabricatorRequestExceptionHandler' => 'aphront/handler/PhabricatorRequestExceptionHandler.php', 'PhabricatorResetPasswordUserLogType' => 'applications/people/userlog/PhabricatorResetPasswordUserLogType.php', 'PhabricatorResourceSite' => 'aphront/site/PhabricatorResourceSite.php', - 'PhabricatorRobotsController' => 'applications/system/controller/PhabricatorRobotsController.php', - 'PhabricatorRobotsPlatformController' => 'applications/system/controller/PhabricatorRobotsPlatformController.php', - 'PhabricatorRobotsResourceController' => 'applications/system/controller/PhabricatorRobotsResourceController.php', + 'PhabricatorRobotsBlogController' => 'applications/system/controller/robots/PhabricatorRobotsBlogController.php', + 'PhabricatorRobotsController' => 'applications/system/controller/robots/PhabricatorRobotsController.php', + 'PhabricatorRobotsPlatformController' => 'applications/system/controller/robots/PhabricatorRobotsPlatformController.php', + 'PhabricatorRobotsResourceController' => 'applications/system/controller/robots/PhabricatorRobotsResourceController.php', + 'PhabricatorRobotsShortController' => 'applications/system/controller/robots/PhabricatorRobotsShortController.php', 'PhabricatorS3FileStorageEngine' => 'applications/files/engine/PhabricatorS3FileStorageEngine.php', 'PhabricatorSMSAuthFactor' => 'applications/auth/factor/PhabricatorSMSAuthFactor.php', 'PhabricatorSQLPatchList' => 'infrastructure/storage/patch/PhabricatorSQLPatchList.php', @@ -11474,9 +11476,11 @@ phutil_register_library_map(array( 'PhabricatorRequestExceptionHandler' => 'AphrontRequestExceptionHandler', 'PhabricatorResetPasswordUserLogType' => 'PhabricatorUserLogType', 'PhabricatorResourceSite' => 'PhabricatorSite', + 'PhabricatorRobotsBlogController' => 'PhabricatorRobotsController', 'PhabricatorRobotsController' => 'PhabricatorController', 'PhabricatorRobotsPlatformController' => 'PhabricatorRobotsController', 'PhabricatorRobotsResourceController' => 'PhabricatorRobotsController', + 'PhabricatorRobotsShortController' => 'PhabricatorRobotsController', 'PhabricatorS3FileStorageEngine' => 'PhabricatorFileStorageEngine', 'PhabricatorSMSAuthFactor' => 'PhabricatorAuthFactor', 'PhabricatorSQLPatchList' => 'Phobject', diff --git a/src/applications/phame/application/PhabricatorPhameApplication.php b/src/applications/phame/application/PhabricatorPhameApplication.php index 9b85a827c1..cd2eb4b487 100644 --- a/src/applications/phame/application/PhabricatorPhameApplication.php +++ b/src/applications/phame/application/PhabricatorPhameApplication.php @@ -70,7 +70,11 @@ final class PhabricatorPhameApplication extends PhabricatorApplication { } public function getBlogRoutes() { - return $this->getLiveRoutes(); + return $this->getLiveRoutes() + array( + '/status/' => 'PhabricatorStatusController', + '/favicon.ico' => 'PhabricatorFaviconController', + '/robots.txt' => 'PhabricatorRobotsBlogController', + ); } private function getLiveRoutes() { diff --git a/src/applications/phurl/application/PhabricatorPhurlApplication.php b/src/applications/phurl/application/PhabricatorPhurlApplication.php index ebff17186b..14ffda77f1 100644 --- a/src/applications/phurl/application/PhabricatorPhurlApplication.php +++ b/src/applications/phurl/application/PhabricatorPhurlApplication.php @@ -55,6 +55,10 @@ final class PhabricatorPhurlApplication extends PhabricatorApplication { public function getShortRoutes() { return array( + '/status/' => 'PhabricatorStatusController', + '/favicon.ico' => 'PhabricatorFaviconController', + '/robots.txt' => 'PhabricatorRobotsShortController', + '/u/(?P[^/]+)' => 'PhabricatorPhurlShortURLController', '.*' => 'PhabricatorPhurlShortURLDefaultController', ); diff --git a/src/applications/system/controller/robots/PhabricatorRobotsBlogController.php b/src/applications/system/controller/robots/PhabricatorRobotsBlogController.php new file mode 100644 index 0000000000..3b26acd6ee --- /dev/null +++ b/src/applications/system/controller/robots/PhabricatorRobotsBlogController.php @@ -0,0 +1,17 @@ +newRobotsRules(); - // Add a small crawl delay (number of seconds between requests) for spiders - // which respect it. The intent here is to prevent spiders from affecting - // performance for users. The possible cost is slower indexing, but that - // seems like a reasonable tradeoff, since most Phabricator installs are - // probably not hugely concerned about cutting-edge SEO. - $out[] = 'Crawl-delay: 1'; - $content = implode("\n", $out)."\n"; return id(new AphrontPlainTextResponse()) diff --git a/src/applications/system/controller/PhabricatorRobotsPlatformController.php b/src/applications/system/controller/robots/PhabricatorRobotsPlatformController.php similarity index 66% rename from src/applications/system/controller/PhabricatorRobotsPlatformController.php rename to src/applications/system/controller/robots/PhabricatorRobotsPlatformController.php index 5132e94e87..b4a3c4fa37 100644 --- a/src/applications/system/controller/PhabricatorRobotsPlatformController.php +++ b/src/applications/system/controller/robots/PhabricatorRobotsPlatformController.php @@ -19,6 +19,13 @@ final class PhabricatorRobotsPlatformController $out[] = 'Disallow: /diffusion/'; $out[] = 'Disallow: /source/'; + // Add a small crawl delay (number of seconds between requests) for spiders + // which respect it. The intent here is to prevent spiders from affecting + // performance for users. The possible cost is slower indexing, but that + // seems like a reasonable tradeoff, since most Phabricator installs are + // probably not hugely concerned about cutting-edge SEO. + $out[] = 'Crawl-delay: 1'; + return $out; } diff --git a/src/applications/system/controller/PhabricatorRobotsResourceController.php b/src/applications/system/controller/robots/PhabricatorRobotsResourceController.php similarity index 90% rename from src/applications/system/controller/PhabricatorRobotsResourceController.php rename to src/applications/system/controller/robots/PhabricatorRobotsResourceController.php index da0db4fe95..367d080c5d 100644 --- a/src/applications/system/controller/PhabricatorRobotsResourceController.php +++ b/src/applications/system/controller/robots/PhabricatorRobotsResourceController.php @@ -10,6 +10,7 @@ final class PhabricatorRobotsResourceController $out[] = 'User-Agent: *'; $out[] = 'Disallow: /'; + $out[] = 'Crawl-delay: 1'; return $out; } diff --git a/src/applications/system/controller/robots/PhabricatorRobotsShortController.php b/src/applications/system/controller/robots/PhabricatorRobotsShortController.php new file mode 100644 index 0000000000..efc906b168 --- /dev/null +++ b/src/applications/system/controller/robots/PhabricatorRobotsShortController.php @@ -0,0 +1,18 @@ +