From 8084acebfaf59ba699143334f0506096df3499f1 Mon Sep 17 00:00:00 2001 From: Yongmin Hong Date: Wed, 3 Jul 2024 23:23:56 +0900 Subject: [PATCH] Phorge(robots.txt): add applebot-extended to disallow Summary: Ref: https://support.apple.com/en-us/119829#datausage Signed-off-by: Yongmin Hong Test Plan: Verify the user-agent matches. Reviewers: O1 revi & automations, revi Reviewed By: O1 revi & automations, revi Differential Revision: https://issuetracker.revi.xyz/D508 --- PhabExt/PhabricatorCustomRobotsTxtController.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/PhabExt/PhabricatorCustomRobotsTxtController.php b/PhabExt/PhabricatorCustomRobotsTxtController.php index d6aadff..e43f6fa 100644 --- a/PhabExt/PhabricatorCustomRobotsTxtController.php +++ b/PhabExt/PhabricatorCustomRobotsTxtController.php @@ -17,7 +17,7 @@ public function processRequest() { // Version timestamp is when I started editing them. // Edit setLastModified at the bottom as well. // Calculate EpochTime via go/epoch - $out[] = '# version: 20240621T160501+0900'; + $out[] = '# version: 20240703T230700+0900'; $out[] = '# also at https://github.com/revi/sandbox.git'; $out[] = 'User-Agent: *'; $out[] = 'Disallow: /diffusion/'; @@ -101,6 +101,10 @@ public function processRequest() { $out[] = '# Ref: https://issuetracker.revi.xyz/u/googleextended'; $out[] = 'User-agent: Google-Extended'; $out[] = 'Disallow: /'; + $out[] = '# Apple AI stuff'; + $out[] = '# Ref: https://support.apple.com/en-us/119829#datausage'; + $out[] = 'User-agent: Applebot-Extended'; + $out[] = 'Disallow: /'; $out[] = '# CCBot (ab)used to train LLMs'; $out[] = '# Ref: https://darkvisitors.com/agents/ccbot'; $out[] = 'User-agent: CCBot'; @@ -180,6 +184,6 @@ public function processRequest() { ->setCacheDurationInSeconds(phutil_units('2 hours in seconds')) // ->setClientIDCookie(false) (Doesn't work /shrug) ->setCanCDN(true) - ->setLastModified(1718953501); + ->setLastModified(1720015620); } }