Classify changesets as "generated" at creation time, in addition to display time

Summary:
Ref T13130. See PHI251. Currently, changesets are marked as "generated" (i.e., the file contains generated code and does not normally need to be reviewed) at display time.

An install would like support for having Owners rules ignore generated files. Additionally, future changes anticipate making "generated" and some other similar behaviors more flexible and more general.

To support these, move toward a world where:

  - Changesets have "attributes": today, generated. In the future, perhaps: third-party, highlight-as, encoding, enormous-text-file, etc.
  - Attributes are either "trusted" (usually: the server assigned the attribute) or "untrusted" (usually: the client assigned the attribute). For attributes like "highlight-as", this isn't relevant, but I'd like to provide tools so that you can't make `arc` mark every file as "generated" and sneak past review rules in the future.

Here, the `differential.generated-paths` config can mark a file as "generated" with a trusted attribute. The `@generated`-in-content rule can mark a file as "generated" with an untrusted attribute.

Putting these attributes on changesets at creation time instead of display time will let Owners interact with changesets cheaply: it won't have to render an entire changeset just to figure out if it's generated or not.

Test Plan:
  - Created a revision touching several files, some generated and some not.
  - Saw the generated files get marked properly with attribute metadata in the database, and show/fold as "Generated" in the UI.

Reviewers: amckinley

Reviewed By: amckinley

Maniphest Tasks: T13130

Differential Revision: https://secure.phabricator.com/D19425

src/applications/differential/parser/DifferentialChangesetParser.php

@@ -542,6 +542,12 @@ final class DifferentialChangesetParser extends Phobject {
     PhutilEventEngine::dispatchEvent($event);
 
     $generated = $event->getValue('is_generated');
+
+    $attribute = $this->changeset->isGeneratedChangeset();
+    if ($attribute) {
+      $generated = true;
+    }
+
     $this->specialAttributes[self::ATTR_GENERATED] = $generated;
   }
 

src/applications/differential/storage/DifferentialChangeset.php

@@ -12,7 +12,7 @@ final class DifferentialChangeset
   protected $awayPaths;
   protected $changeType;
   protected $fileType;
-  protected $metadata;
+  protected $metadata = array();
   protected $oldProperties;
   protected $newProperties;
   protected $addLines;
@@ -24,6 +24,11 @@ final class DifferentialChangeset
 
   const TABLE_CACHE = 'differential_changeset_parse_cache';
 
+  const METADATA_TRUSTED_ATTRIBUTES = 'attributes.trusted';
+  const METADATA_UNTRUSTED_ATTRIBUTES = 'attributes.untrusted';
+
+  const ATTRIBUTE_GENERATED = 'generated';
+
   protected function getConfiguration() {
     return array(
       self::CONFIG_SERIALIZATION => array(
@@ -266,6 +271,90 @@ final class DifferentialChangeset
     return null;
   }
 
+  public function setChangesetMetadata($key, $value) {
+    if (!is_array($this->metadata)) {
+      $this->metadata = array();
+    }
+
+    $this->metadata[$key] = $value;
+
+    return $this;
+  }
+
+  public function getChangesetMetadata($key, $default = null) {
+    if (!is_array($this->metadata)) {
+      return $default;
+    }
+
+    return idx($this->metadata, $key, $default);
+  }
+
+  private function setInternalChangesetAttribute($trusted, $key, $value) {
+    if ($trusted) {
+      $meta_key = self::METADATA_TRUSTED_ATTRIBUTES;
+    } else {
+      $meta_key = self::METADATA_UNTRUSTED_ATTRIBUTES;
+    }
+
+    $attributes = $this->getChangesetMetadata($meta_key, array());
+    $attributes[$key] = $value;
+    $this->setChangesetMetadata($meta_key, $attributes);
+
+    return $this;
+  }
+
+  private function getInternalChangesetAttributes($trusted) {
+    if ($trusted) {
+      $meta_key = self::METADATA_TRUSTED_ATTRIBUTES;
+    } else {
+      $meta_key = self::METADATA_UNTRUSTED_ATTRIBUTES;
+    }
+
+    return $this->getChangesetMetadata($meta_key, array());
+  }
+
+  public function setTrustedChangesetAttribute($key, $value) {
+    return $this->setInternalChangesetAttribute(true, $key, $value);
+  }
+
+  public function getTrustedChangesetAttributes() {
+    return $this->getInternalChangesetAttributes(true);
+  }
+
+  public function getTrustedChangesetAttribute($key, $default = null) {
+    $map = $this->getTrustedChangesetAttributes();
+    return idx($map, $key, $default);
+  }
+
+  public function setUntrustedChangesetAttribute($key, $value) {
+    return $this->setInternalChangesetAttribute(false, $key, $value);
+  }
+
+  public function getUntrustedChangesetAttributes() {
+    return $this->getInternalChangesetAttributes(false);
+  }
+
+  public function getUntrustedChangesetAttribute($key, $default = null) {
+    $map = $this->getUntrustedChangesetAttributes();
+    return idx($map, $key, $default);
+  }
+
+  public function getChangesetAttributes() {
+    // Prefer trusted values over untrusted values when both exist.
+    return
+      $this->getTrustedChangesetAttributes() +
+      $this->getUntrustedChangesetAttributes();
+  }
+
+  public function getChangesetAttribute($key, $default = null) {
+    $map = $this->getChangesetAttributes();
+    return idx($map, $key, $default);
+  }
+
+  public function isGeneratedChangeset() {
+    return $this->getChangesetAttribute(self::ATTRIBUTE_GENERATED);
+  }
+
 
 /* -(  PhabricatorPolicyInterface  )----------------------------------------- */
 

src/applications/differential/storage/DifferentialDiff.php

@@ -226,6 +226,8 @@ final class DifferentialDiff
       $changeset->setAddLines($add_lines);
       $changeset->setDelLines($del_lines);
 
+      self::detectGeneratedCode($changeset);
+
       $diff->addUnsavedChangeset($changeset);
     }
     $diff->setLineCount($lines);
@@ -821,5 +823,48 @@ final class DifferentialDiff
     );
   }
 
+  private static function detectGeneratedCode(
+    DifferentialChangeset $changeset) {
+
+    $is_generated_trusted = self::isTrustedGeneratedCode($changeset);
+
+    $changeset->setTrustedChangesetAttribute(
+      DifferentialChangeset::ATTRIBUTE_GENERATED,
+      $is_generated_trusted);
+
+    $is_generated_untrusted = self::isUntrustedGeneratedCode($changeset);
+
+    $changeset->setUntrustedChangesetAttribute(
+      DifferentialChangeset::ATTRIBUTE_GENERATED,
+      $is_generated_untrusted);
+  }
+
+  private static function isTrustedGeneratedCode(
+    DifferentialChangeset $changeset) {
+
+    $filename = $changeset->getFilename();
+
+    $paths = PhabricatorEnv::getEnvConfig('differential.generated-paths');
+    foreach ($paths as $regexp) {
+      if (preg_match($regexp, $filename)) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  private static function isUntrustedGeneratedCode(
+    DifferentialChangeset $changeset) {
+
+    if ($changeset->getHunks()) {
+      $new_data = $changeset->makeNewFile();
+      if (strpos($new_data, '@'.'generated') !== false) {
+        return true;
+      }
+    }
+
+    return false;
+  }
 
 }