Robots.txt: f off PetalBot
Summary: Is on .xyz/robots.txt blacklist. add it here as well. Signed-off-by: Yongmin Hong <revi@omglol.email> Test Plan: Deploy Reviewers: O1 revi & automations, revi Reviewed By: O1 revi & automations, revi Differential Revision: https://issuetracker.revi.xyz/D125
This commit is contained in:
parent
86751ae2ea
commit
bf5c8a9281
2 changed files with 9 additions and 7 deletions
7
.arclint
7
.arclint
|
@ -6,9 +6,6 @@
|
||||||
"chmod": {
|
"chmod": {
|
||||||
"type": "chmod"
|
"type": "chmod"
|
||||||
},
|
},
|
||||||
"filename": {
|
|
||||||
"type": "filename"
|
|
||||||
},
|
|
||||||
"json": {
|
"json": {
|
||||||
"type": "json",
|
"type": "json",
|
||||||
"include": [
|
"include": [
|
||||||
|
@ -34,7 +31,9 @@
|
||||||
"type": "text",
|
"type": "text",
|
||||||
"text.max-line-length": 80,
|
"text.max-line-length": 80,
|
||||||
"severity": {
|
"severity": {
|
||||||
"2": "advice"
|
"1": "advice",
|
||||||
|
"2": "advice",
|
||||||
|
"5": "disabled"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,7 +19,7 @@ public function processRequest() {
|
||||||
// Version timestamp is when I started editing them.
|
// Version timestamp is when I started editing them.
|
||||||
// Edit setLastModified at the bottom as well.
|
// Edit setLastModified at the bottom as well.
|
||||||
// Calculate EpochTime via go/epoch
|
// Calculate EpochTime via go/epoch
|
||||||
$out[] = '# version: 20240511T052727+0900';
|
$out[] = '# version: 20240523T040800+0900';
|
||||||
$out[] = '# also at https://github.com/revi/sandbox.git';
|
$out[] = '# also at https://github.com/revi/sandbox.git';
|
||||||
$out[] = 'User-Agent: *';
|
$out[] = 'User-Agent: *';
|
||||||
$out[] = 'Disallow: /diffusion/';
|
$out[] = 'Disallow: /diffusion/';
|
||||||
|
@ -30,7 +30,7 @@ public function processRequest() {
|
||||||
$out[] = 'Disallow: /login';
|
$out[] = 'Disallow: /login';
|
||||||
$out[] = 'Disallow: /maniphest/transaction';
|
$out[] = 'Disallow: /maniphest/transaction';
|
||||||
$out[] = 'Disallow: /tag';
|
$out[] = 'Disallow: /tag';
|
||||||
$out[] = 'Disallow: /search/query/all';
|
$out[] = 'Disallow: /search/';
|
||||||
$out[] = 'Disallow: /conduit';
|
$out[] = 'Disallow: /conduit';
|
||||||
$out[] = 'Disallow: /api';
|
$out[] = 'Disallow: /api';
|
||||||
$out[] = 'Disallow: /project';
|
$out[] = 'Disallow: /project';
|
||||||
|
@ -98,6 +98,9 @@ public function processRequest() {
|
||||||
$out[] = '# Ref: https://darkvisitors.com/agents/bytespider';
|
$out[] = '# Ref: https://darkvisitors.com/agents/bytespider';
|
||||||
$out[] = 'User-agent: Bytespider';
|
$out[] = 'User-agent: Bytespider';
|
||||||
$out[] = 'Disallow: /';
|
$out[] = 'Disallow: /';
|
||||||
|
$out[] = '# Block PetalBot, misbehaving';
|
||||||
|
$out[] = 'User-agent: PetalBot';
|
||||||
|
$out[] = 'Disallow: /';
|
||||||
// Crawl-delay entries at the bottom
|
// Crawl-delay entries at the bottom
|
||||||
// Ref: https://github.com/otwcode/otwarchive/pull/4411#discussion_r1044351129
|
// Ref: https://github.com/otwcode/otwarchive/pull/4411#discussion_r1044351129
|
||||||
$out[] = 'User-agent: *';
|
$out[] = 'User-agent: *';
|
||||||
|
@ -109,6 +112,6 @@ public function processRequest() {
|
||||||
->setContent($content)
|
->setContent($content)
|
||||||
->setCacheDurationInSeconds(phutil_units('2 hours in seconds'))
|
->setCacheDurationInSeconds(phutil_units('2 hours in seconds'))
|
||||||
->setCanCDN(true)
|
->setCanCDN(true)
|
||||||
->setLastModified(1715372847);
|
->setLastModified(1716404880);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue