revi/sandbox
1
0
Fork 0
Code Issues Wiki Activity

RobotsTxt: add more rules

Browse source 
Interest of nobody.

Bug: N/A
Signed-off-by: Yongmin Hong <revi@omglol.email>
This commit is contained in:
revi 2024-04-17 01:35:10 +09:00
parent c4eb4aba4e
commit cb20e4d795
Signed by: revi
GPG key ID: 1EB4F6CEEA100E94
1 changed files with 15 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
PhabExt/PhabricatorCustomRobotsTxtController.php
View file

@ -10,7 +10,8 @@ public function processRequest() {
$out = array(); $out = array();
$out[] = '# Forked from phabricator.wikimedia.org, we.phorge.it'; $out[] = '# Forked from phabricator.wikimedia.org, we.phorge.it';
$out[] = '# version: 20240416T211100+0900'; // Version timestamp is when I started editing them.
$out[] = '# version: 20240417T011800+0900';
$out[] = '# also at https://github.com/revi/sandbox.git'; $out[] = '# also at https://github.com/revi/sandbox.git';
$out[] = 'User-Agent: *'; $out[] = 'User-Agent: *';
$out[] = 'Disallow: /diffusion/'; $out[] = 'Disallow: /diffusion/';
@ -33,8 +34,19 @@ public function processRequest() {
$out[] = 'Disallow: /dashboard'; $out[] = 'Disallow: /dashboard';
$out[] = 'Disallow: /calendar'; $out[] = 'Disallow: /calendar';
$out[] = 'Disallow: /herald'; $out[] = 'Disallow: /herald';
// This is commits.
$out[] = 'Disallow: /r*'; $out[] = 'Disallow: /r*';
// This is pastes (P$)
$out[] = 'Disallow: /P*%24*'; $out[] = 'Disallow: /P*%24*';
$out[] = 'Disallow: /phame';
// This is blog entries (J$)
$out[] = 'Disallow: J*%24*';
// This is user list.
// As of 2024-04-17 user list is behind auth but who knows it might change?
$out[] = '/people';
// This is user profile link.
$out[] = 'Disallow: /p/';
// Phorge specific entries end here.
$out[] = '# This is cloudflare endpoint'; $out[] = '# This is cloudflare endpoint';
$out[] = '# Ref: https://developers.cloudflare.com/fundamentals/reference/cdn-cgi-endpoint/'; $out[] = '# Ref: https://developers.cloudflare.com/fundamentals/reference/cdn-cgi-endpoint/';
$out[] = 'Disallow: /cdn-cgi/'; $out[] = 'Disallow: /cdn-cgi/';
@ -55,6 +67,8 @@ public function processRequest() {
$out[] = '# Ref: https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers?hl=en#google-extended'; $out[] = '# Ref: https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers?hl=en#google-extended';
$out[] = 'User-agent: Google-Extended'; $out[] = 'User-agent: Google-Extended';
$out[] = 'Disallow: /'; $out[] = 'Disallow: /';
// Crawl-delay entries at the bottom
// Ref: https://github.com/otwcode/otwarchive/pull/4411#discussion_r1044351129
$out[] = 'User-agent: *'; $out[] = 'User-agent: *';
$out[] = 'Crawl-delay: 1'; $out[] = 'Crawl-delay: 1';