From 70cbb7ccd8e038066e3c3d13ee98ab9c1a0ca599 Mon Sep 17 00:00:00 2001
From: Yongmin Hong <revi@omglol.email>
Date: Tue, 16 Apr 2024 21:04:16 +0900
Subject: [PATCH] phab-ext: test

Signed-off-by: Yongmin Hong <revi@omglol.email>
---
 .../PhabricatorCustomRobotsTxtController.php  | 52 +++++++++++++++++--
 1 file changed, 47 insertions(+), 5 deletions(-)

diff --git a/PhabExt/PhabricatorCustomRobotsTxtController.php b/PhabExt/PhabricatorCustomRobotsTxtController.php
index a2a3e0d..917ecc7 100644
--- a/PhabExt/PhabricatorCustomRobotsTxtController.php
+++ b/PhabExt/PhabricatorCustomRobotsTxtController.php
@@ -1,6 +1,6 @@
 <?php
 
-final class PhabricatorRobotsController extends PhabricatorController {
+abstract class PhabricatorRobotsController extends PhabricatorController {
 
   public function shouldRequireLogin() {
     return false;
@@ -9,11 +9,53 @@ public function shouldRequireLogin() {
   public function processRequest() {
     $out = array();
 
+    $out[] = '# Forked from phabricator.wikimedia, we.phorge.it';
+    $out[] = '# version: 20240416T203000+0900';
+    $out[] = '# also at https://bugs.revi.xyz/P49';
     $out[] = 'User-Agent: *';
-    $out[] = 'Disallow: /herald/';
-    $out[] = 'Disallow: /passphrase/';
-    $out[] = 'Disallow: /conduit/';
-    $out[] = 'Disallow: /dashboard/';
+    $out[] = 'Disallow: /diffusion/';
+    $out[] = 'Disallow: /source/';
+    $out[] = 'Disallow: /multimeter/';
+    $out[] = 'Disallow: /file/';
+    $out[] = 'Disallow: /project/ sprint';
+    $out[] = 'Disallow: /policy/explain';
+    $out[] = 'Disallow: /auth';
+    $out[] = 'Disallow: /login';
+    $out[] = 'Disallow: /maniphest/transaction';
+    $out[] = 'Disallow: /tag';
+    $out[] = 'Disallow: /search/query/all';
+    $out[] = 'Disallow: /conduit';
+    $out[] = 'Disallow: /api';
+    $out[] = 'Disallow: /project';
+    $out[] = 'Disallow: /applications';
+    $out[] = 'Disallow: /token';
+    $out[] = 'Disallow: /pholio';
+    $out[] = 'Disallow: /dashboard';
+    $out[] = 'Disallow: /calendar';
+    $out[] = 'Disallow: /herald';
+    $out[] = 'Disallow: /r*';
+    $out[] = 'Disallow: /P*%24*';
+    $out[] = '# This is cloudflare endpoint';
+    $out[] = 'https://developers.cloudflare.com/fundamentals/reference/cdn-cgi-endpoint/';
+    $out[] = 'Disallow: /cdn-cgi/';
+    $out[] = '# Google Ads are not welcome';
+    $out[] = 'User-agent: Mediapartners-Google';
+    $out[] = 'Disallow: /';
+    $out[] = 'User-agent: AdsBot-Google';
+    $out[] = 'Disallow: /';
+    $out[] = 'User-agent: AdsBot-Google-Mobile';
+    $out[] = 'Disallow: /';
+    $out[] = '# ChatGPT Crawlers are not welcome';
+    $out[] = '# Ref: https://platform.openai.com/docs/plugins/bot';
+    $out[] = 'User-agent: ChatGPT-User';
+    $out[] = 'Disallow: /';
+    $out[] = 'User-agent: GPTBot';
+    $out[] = 'Disallow: /';
+    $out[] = '# Google Gemini AI Crawlers are also not welcome';
+    $out[] = '# Ref: https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers?hl=en#google-extended';
+    $out[] = 'User-agent: Google-Extended';
+    $out[] = 'Disallow: /';
+    $out[] = 'User-agent: *';
     $out[] = 'Crawl-delay: 1';
 
     $content = implode("\n", $out)."\n";