Cache results of repository linters

Summary:
The cache key is repository base revision and hash of all modified files.
It isn't perfect in SVN where every file might have a different revision.
It's also suboptimal as just committing or amending changes the cache key even if the files contents weren't modified.
We can improve it later, perhaps by using previous revision and files modified since it.

Test Plan:
Changed granularity of XHPAST linter to repository.
Linted the same repo twice, verified that it was read from cache the second file.
Changed a single file, verified that all files were re-linted.

Reviewers: epriestley

Reviewed By: epriestley

CC: aran, Korvin

Differential Revision: https://secure.phabricator.com/D4608

src/lint/engine/ArcanistLintEngine.php

@@ -52,6 +52,7 @@ abstract class ArcanistLintEngine {
   protected $lineToFirstChar = array();
   private $cachedResults;
   private $cacheVersion;
+  private $repositoryVersion;
   private $results = array();
   private $stopped = array();
   private $minimumSeverity = ArcanistLintSeverity::SEVERITY_DISABLED;
@@ -212,13 +213,26 @@ abstract class ArcanistLintEngine {
             unset($paths[$key]);
           }
           if (isset($this->cachedResults[$path][$this->cacheVersion])) {
-            if ($cache_granularity == ArcanistLinter::GRANULARITY_FILE) {
+            $cached_result = $this->cachedResults[$path][$this->cacheVersion];
+
+            switch ($cache_granularity) {
+              case ArcanistLinter::GRANULARITY_FILE:
+                $use_cache = true;
+                break;
+              case ArcanistLinter::GRANULARITY_DIRECTORY:
+              case ArcanistLinter::GRANULARITY_REPOSITORY:
+                $repository_version = idx($cached_result, 'repository_version');
+                $use_cache = ($this->repositoryVersion == $repository_version);
+                break;
+              default:
+                $use_cache = false;
+                break;
+            }
+
+            if ($use_cache) {
               unset($paths[$key]);
 
-              $cached_stopped = idx(
-                $this->cachedResults[$path][$this->cacheVersion],
-                'stopped');
-              if ($cached_stopped == $linter_name) {
+              if (idx($cached_result, 'stopped') == $linter_name) {
                 $this->stopped[$path] = $linter_name;
               }
             }
@@ -254,7 +268,7 @@ abstract class ArcanistLintEngine {
         if (!$this->isRelevantMessage($message)) {
           continue;
         }
-        if ($cache_granularity != ArcanistLinter::GRANULARITY_FILE) {
+        if ($cache_granularity == ArcanistLinter::GRANULARITY_GLOBAL) {
           $message->setUncacheable(true);
         }
         $result = $this->getResultForPath($message->getPath());
@@ -266,6 +280,7 @@ abstract class ArcanistLintEngine {
       foreach ($this->cachedResults as $path => $messages) {
         $messages = idx($messages, $this->cacheVersion, array());
         unset($messages['stopped']);
+        unset($messages['repository_version']);
         foreach ($messages as $message) {
           $this->getResultForPath($path)->addMessage(
             ArcanistLintMessage::newFromDictionary($message));
@@ -327,6 +342,11 @@ abstract class ArcanistLintEngine {
     }
   }
 
+  public function setRepositoryVersion($version) {
+    $this->repositoryVersion = $version;
+    return $this;
+  }
+
   private function isRelevantMessage(ArcanistLintMessage $message) {
     // When a user runs "arc lint", we default to raising only warnings on
     // lines they have changed (errors are still raised anywhere in the

src/workflow/ArcanistBaseWorkflow.php

@@ -65,6 +65,7 @@ abstract class ArcanistBaseWorkflow extends Phobject {
   private $arcanistConfiguration;
   private $parentWorkflow;
   private $workingDirectory;
+  private $repositoryVersion;
 
   private $changeCache = array();
 
@@ -1538,4 +1539,18 @@ abstract class ArcanistBaseWorkflow extends Phobject {
     return $this;
   }
 
+  protected function getRepositoryVersion() {
+    if (!$this->repositoryVersion) {
+      $api = $this->getRepositoryAPI();
+      $versions = array('' => $api->getSourceControlBaseRevision());
+      foreach ($api->getUncommittedStatus() as $path => $mask) {
+        $versions[$path] = (Filesystem::pathExists($path)
+          ? md5_file($path)
+          : '');
+      }
+      $this->repositoryVersion = md5(json_encode($versions));
+    }
+    return $this->repositoryVersion;
+  }
+
 }

src/workflow/ArcanistLintWorkflow.php

@@ -195,6 +195,7 @@ EOTEXT
     $engine = newv($engine, array());
     $this->engine = $engine;
     $engine->setWorkingCopy($working_copy);
+    $engine->setRepositoryVersion($this->getRepositoryVersion());
 
     $engine->setMinimumSeverity(
       $this->getArgument('severity', self::DEFAULT_SEVERITY));
@@ -517,18 +518,20 @@ EOTEXT
         }
         $hash = md5_file($abs_path);
         $version = $result->getCacheVersion();
-        $cached[$path] = array($hash => array($version => array()));
+        $cached_path = array();
         if (isset($stopped[$path])) {
-          $cached[$path][$hash][$version]['stopped'] = $stopped[$path];
+          $cached_path['stopped'] = $stopped[$path];
         }
+        $cached_path['repository_version'] = $this->getRepositoryVersion();
         foreach ($result->getMessages() as $message) {
           if ($message->isUncacheable()) {
             continue;
           }
           if (!$message->isPatchApplied()) {
-            $cached[$path][$hash][$version][] = $message->toDictionary();
+            $cached_path[] = $message->toDictionary();
           }
         }
+        $cached[$path] = array($hash => array($version => $cached_path));
       }
       $cache[$this->getCacheKey()] = $cached;
       // TODO: Garbage collection.