phabExt: add lastModified time and few more rules

Should figure out how to disable set-cookie to enable Cloudflare
cache...

Bug: N/A
Signed-off-by: Yongmin Hong <revi@omglol.email>

PhabExt/PhabricatorCustomRobotsTxtController.php

@@ -11,7 +11,9 @@ public function processRequest() {
 
     $out[] = '# Forked from phabricator.wikimedia.org, we.phorge.it';
     // Version timestamp is when I started editing them.
-    $out[] = '# version: 20240421T042000+0900';
+    // Edit setLastModified at the bottom as well.
+    // Calculate EpochTime via go/ZoneStamp
+    $out[] = '# version: 20240429T175800+0900';
     $out[] = '# also at https://github.com/revi/sandbox.git';
     $out[] = 'User-Agent: *';
     $out[] = 'Disallow: /diffusion/';
@@ -35,10 +37,14 @@ public function processRequest() {
     $out[] = 'Disallow: /herald';
     // This is commits.
     $out[] = 'Disallow: /r*';
+    // This is Files. (F$)
+    $out[] = 'Disallow: /file';
+    $out[] = 'Disallow: /F*%24*';
     // This is pastes (P$)
+    $out[] = 'Disallow: /paste';
     $out[] = 'Disallow: /P*%24*';
-    $out[] = 'Disallow: /phame';
     // This is blog entries (J$)
+    $out[] = 'Disallow: /phame';
     $out[] = 'Disallow: /J*%24*';
     // This is user list.
     // As of 2024-04-17 user list is behind auth but who knows it might change?
@@ -94,6 +100,7 @@ public function processRequest() {
     return id(new AphrontPlainTextResponse())
       ->setContent($content)
       ->setCacheDurationInSeconds(phutil_units('2 hours in seconds'))
-      ->setCanCDN(true);
+      ->setCanCDN(true)
+      ->setLastModified(1714381080);
   }
 }