From bf5c8a9281508b0517eb64d1a942168e496fbd66 Mon Sep 17 00:00:00 2001
From: Yongmin Hong <revi@omglol.email>
Date: Wed, 22 May 2024 19:11:36 +0000
Subject: [PATCH] Robots.txt: f off PetalBot

Summary:
Is on .xyz/robots.txt blacklist. add it here as well.

Signed-off-by: Yongmin Hong <revi@omglol.email>

Test Plan: Deploy

Reviewers: O1 revi & automations, revi

Reviewed By: O1 revi & automations, revi

Differential Revision: https://issuetracker.revi.xyz/D125
---
 .arclint                                         | 7 +++----
 PhabExt/PhabricatorCustomRobotsTxtController.php | 9 ++++++---
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/.arclint b/.arclint
index f5b009b..e445eac 100644
--- a/.arclint
+++ b/.arclint
@@ -6,9 +6,6 @@
     "chmod": {
       "type": "chmod"
     },
-    "filename": {
-      "type": "filename"
-    },
     "json": {
       "type": "json",
       "include": [
@@ -34,7 +31,9 @@
       "type": "text",
       "text.max-line-length": 80,
       "severity": {
-        "2": "advice"
+        "1": "advice",
+        "2": "advice",
+        "5": "disabled"
       }
     }
   }
diff --git a/PhabExt/PhabricatorCustomRobotsTxtController.php b/PhabExt/PhabricatorCustomRobotsTxtController.php
index a72ae75..203290d 100644
--- a/PhabExt/PhabricatorCustomRobotsTxtController.php
+++ b/PhabExt/PhabricatorCustomRobotsTxtController.php
@@ -19,7 +19,7 @@ public function processRequest() {
 		// Version timestamp is when I started editing them.
 		// Edit setLastModified at the bottom as well.
 		// Calculate EpochTime via go/epoch
-		$out[] = '# version: 20240511T052727+0900';
+		$out[] = '# version: 20240523T040800+0900';
 		$out[] = '# also at https://github.com/revi/sandbox.git';
 		$out[] = 'User-Agent: *';
 		$out[] = 'Disallow: /diffusion/';
@@ -30,7 +30,7 @@ public function processRequest() {
 		$out[] = 'Disallow: /login';
 		$out[] = 'Disallow: /maniphest/transaction';
 		$out[] = 'Disallow: /tag';
-		$out[] = 'Disallow: /search/query/all';
+		$out[] = 'Disallow: /search/';
 		$out[] = 'Disallow: /conduit';
 		$out[] = 'Disallow: /api';
 		$out[] = 'Disallow: /project';
@@ -98,6 +98,9 @@ public function processRequest() {
 		$out[] = '# Ref: https://darkvisitors.com/agents/bytespider';
 		$out[] = 'User-agent: Bytespider';
 		$out[] = 'Disallow: /';
+		$out[] = '# Block PetalBot, misbehaving';
+		$out[] = 'User-agent: PetalBot';
+		$out[] = 'Disallow: /';
 		// Crawl-delay entries at the bottom
 		// Ref: https://github.com/otwcode/otwarchive/pull/4411#discussion_r1044351129
 		$out[] = 'User-agent: *';
@@ -109,6 +112,6 @@ public function processRequest() {
 			->setContent($content)
 			->setCacheDurationInSeconds(phutil_units('2 hours in seconds'))
 			->setCanCDN(true)
-			->setLastModified(1715372847);
+			->setLastModified(1716404880);
 	}
 }