revi/sandbox
1
0
Fork 0
Code Issues Wiki Activity

Robots.txt: f off PetalBot

Browse source 
Summary:
Is on .xyz/robots.txt blacklist. add it here as well.

Signed-off-by: Yongmin Hong <revi@omglol.email>

Test Plan: Deploy

Reviewers: O1 revi & automations, revi

Reviewed By: O1 revi & automations, revi

Differential Revision: https://issuetracker.revi.xyz/D125
This commit is contained in:
revi 2024-05-22 19:11:36 +00:00
parent 86751ae2ea
commit bf5c8a9281
Signed by: revi
GPG key ID: 1EB4F6CEEA100E94
2 changed files with 9 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.arclint
View file

@ -6,9 +6,6 @@
"chmod": { "chmod": {
"type": "chmod" "type": "chmod"
}, },
"filename": {
"type": "filename"
},
"json": { "json": {
"type": "json", "type": "json",
"include": [ "include": [
@ -34,7 +31,9 @@
"type": "text", "type": "text",
"text.max-line-length": 80, "text.max-line-length": 80,
"severity": { "severity": {
"2": "advice" "1": "advice",
"2": "advice",
"5": "disabled"
} }
} }
} }

9
PhabExt/PhabricatorCustomRobotsTxtController.php
View file

@ -19,7 +19,7 @@ public function processRequest() {
// Version timestamp is when I started editing them. // Version timestamp is when I started editing them.
// Edit setLastModified at the bottom as well. // Edit setLastModified at the bottom as well.
// Calculate EpochTime via go/epoch // Calculate EpochTime via go/epoch
$out[] = '# version: 20240511T052727+0900'; $out[] = '# version: 20240523T040800+0900';
$out[] = '# also at https://github.com/revi/sandbox.git'; $out[] = '# also at https://github.com/revi/sandbox.git';
$out[] = 'User-Agent: *'; $out[] = 'User-Agent: *';
$out[] = 'Disallow: /diffusion/'; $out[] = 'Disallow: /diffusion/';
@ -30,7 +30,7 @@ public function processRequest() {
$out[] = 'Disallow: /login'; $out[] = 'Disallow: /login';
$out[] = 'Disallow: /maniphest/transaction'; $out[] = 'Disallow: /maniphest/transaction';
$out[] = 'Disallow: /tag'; $out[] = 'Disallow: /tag';
$out[] = 'Disallow: /search/query/all'; $out[] = 'Disallow: /search/';
$out[] = 'Disallow: /conduit'; $out[] = 'Disallow: /conduit';
$out[] = 'Disallow: /api'; $out[] = 'Disallow: /api';
$out[] = 'Disallow: /project'; $out[] = 'Disallow: /project';
@ -98,6 +98,9 @@ public function processRequest() {
$out[] = '# Ref: https://darkvisitors.com/agents/bytespider'; $out[] = '# Ref: https://darkvisitors.com/agents/bytespider';
$out[] = 'User-agent: Bytespider'; $out[] = 'User-agent: Bytespider';
$out[] = 'Disallow: /'; $out[] = 'Disallow: /';
$out[] = '# Block PetalBot, misbehaving';
$out[] = 'User-agent: PetalBot';
$out[] = 'Disallow: /';
// Crawl-delay entries at the bottom // Crawl-delay entries at the bottom
// Ref: https://github.com/otwcode/otwarchive/pull/4411#discussion_r1044351129 // Ref: https://github.com/otwcode/otwarchive/pull/4411#discussion_r1044351129
$out[] = 'User-agent: *'; $out[] = 'User-agent: *';
@ -109,6 +112,6 @@ public function processRequest() {
->setContent($content) ->setContent($content)
->setCacheDurationInSeconds(phutil_units('2 hours in seconds')) ->setCacheDurationInSeconds(phutil_units('2 hours in seconds'))
->setCanCDN(true) ->setCanCDN(true)
->setLastModified(1715372847); ->setLastModified(1716404880);
} }
} }