revi/sandbox
1
0
Fork 0
Code Issues Wiki Activity

robotstxt: just use *

Browse source 
Summary:
Fixes T138. Seems like Google doesn't care.

Signed-off-by: Yongmin Hong <revi@omglol.email>

Test Plan: Try http://go/google/search w/ updated robots.txt entries

Reviewers: O1 revi & automations, revi

Reviewed By: O1 revi & automations, revi

Maniphest Tasks: T138

Differential Revision: https://issuetracker.revi.xyz/D75
This commit is contained in:
revi 2024-05-11 17:28:38 +09:00
parent a257d3ccb4
commit 467eb9b2d1
Signed by: revi
GPG key ID: 1EB4F6CEEA100E94
2 changed files with 103 additions and 123 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
.editorconfig
View file

@ -5,32 +5,12 @@ root = true
[*] [*]
charset = utf-8 charset = utf-8
end_of_line = lf end_of_line = lf
indent_size = tab indent_size = 2
indent_style = tab indent_style = space
insert_final_newline = true insert_final_newline = true
tab_width = 4
trim_trailing_whitespace = true trim_trailing_whitespace = true
[*.md] [.php]
indent_style = space # Use tabs for php file indentation
indent_size = 2 indent_style = tab
tab_width = 4
[.arc*]
indent_size = 2
indent_style = space
# Tabs may not be valid YAML
# @see https://yaml.org/spec/1.2/spec.html#id2777534
[*.{css,js,json,yml,yaml}]
indent_size = 2
indent_style = space
# yaml-lint configuration
# YAML files should be indented with spaces, which .yamllint is
[.yamllint]
indent_size = 2
indent_style = space
[.git/**]
indent_size = 2
indent_style = space

18
PhabExt/PhabricatorCustomRobotsTxtController.php
View file

@ -19,7 +19,7 @@ public function processRequest() {
// Version timestamp is when I started editing them. // Version timestamp is when I started editing them.
// Edit setLastModified at the bottom as well. // Edit setLastModified at the bottom as well.
// Calculate EpochTime via go/epoch // Calculate EpochTime via go/epoch
$out[] = '# version: 20240509T235513+0900'; $out[] = '# version: 20240511T052727+0900';
$out[] = '# also at https://github.com/revi/sandbox.git'; $out[] = '# also at https://github.com/revi/sandbox.git';
$out[] = 'User-Agent: *'; $out[] = 'User-Agent: *';
$out[] = 'Disallow: /diffusion/'; $out[] = 'Disallow: /diffusion/';
@ -42,18 +42,18 @@ public function processRequest() {
$out[] = 'Disallow: /herald'; $out[] = 'Disallow: /herald';
// This is commits. // This is commits.
$out[] = 'Disallow: /r*'; $out[] = 'Disallow: /r*';
// This is differential reviews // This is differential revisions. (D*)
$out[] = 'Disallow: /differential'; $out[] = 'Disallow: /differential';
$out[] = 'Disallow: /D*%24*'; $out[] = 'Disallow: /D*';
// This is Files. (F$) // This is Files. (F*)
$out[] = 'Disallow: /file'; $out[] = 'Disallow: /file';
$out[] = 'Disallow: /F*%24*'; $out[] = 'Disallow: /F*';
// This is pastes (P$) // This is pastes (P*)
$out[] = 'Disallow: /paste'; $out[] = 'Disallow: /paste';
$out[] = 'Disallow: /P*%24*'; $out[] = 'Disallow: /P*';
// This is blog entries (J$) // This is blog entries (J$)
$out[] = 'Disallow: /phame'; $out[] = 'Disallow: /phame';
$out[] = 'Disallow: /J*%24*'; $out[] = 'Disallow: /J*';
// This is user list. // This is user list.
// As of 2024-04-17 user list is behind auth but who knows it might change? // As of 2024-04-17 user list is behind auth but who knows it might change?
$out[] = 'Disallow: /people'; $out[] = 'Disallow: /people';
@ -109,6 +109,6 @@ public function processRequest() {
->setContent($content) ->setContent($content)
->setCacheDurationInSeconds(phutil_units('2 hours in seconds')) ->setCacheDurationInSeconds(phutil_units('2 hours in seconds'))
->setCanCDN(true) ->setCanCDN(true)
->setLastModified(1715266513); ->setLastModified(1715372847);
} }
} }