Allow Diviner groups to be configured in .book files

Summary:
Ref T988. Currently, every class/function needs to be annotated with `@group`, but 99% of this data can be inferred from file structure, at least in this project. Allow group specifications like:

  "paste" : {
    "name" : "Paste",
    "include" : "(^src/applications/paste/)"
  }

..to automatically put everything defined there in the "paste" group. A list of regexps is also supported. Depends on D6855.

Test Plan: Regenerated documentation with `bin/diviner generate --book src/docs/book/phabricator.book --clean`, observed all Paste stuff go in the paste group.

Reviewers: btrahan

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T988

Differential Revision: https://secure.phabricator.com/D6856

src/applications/diviner/atom/DivinerAtom.php

@@ -295,10 +295,8 @@ final class DivinerAtom {
   }
 
   public function getRef() {
-    $group = null;
     $title = null;
     if ($this->docblockMeta) {
-      $group = $this->getDocblockMetaValue('group');
       $title = $this->getDocblockMetaValue('title');
     }
 
@@ -308,7 +306,7 @@ final class DivinerAtom {
       ->setType($this->getType())
       ->setName($this->getName())
       ->setTitle($title)
-      ->setGroup($group);
+      ->setGroup($this->getProperty('group'));
   }
 
   public static function newFromDictionary(array $dictionary) {

src/applications/diviner/atomizer/DivinerAtomizer.php

@@ -7,6 +7,7 @@ abstract class DivinerAtomizer {
 
   private $book;
   private $fileName;
+  private $atomContext;
 
   /**
    * If you make a significant change to an atomizer, you can bump this
@@ -16,9 +17,33 @@ abstract class DivinerAtomizer {
     return 1;
   }
 
-  final public function atomize($file_name, $file_data) {
+  final public function atomize($file_name, $file_data, array $context) {
     $this->fileName = $file_name;
-    return $this->executeAtomize($file_name, $file_data);
+    $this->atomContext = $context;
+    $atoms = $this->executeAtomize($file_name, $file_data);
+
+    // Promote the "@group" special to a property. If there's no "@group" on
+    // an atom but the file it's in matches a group pattern, associate it with
+    // the right group.
+    foreach ($atoms as $atom) {
+      $group = null;
+      try {
+        $group = $atom->getDocblockMetaValue('group');
+      } catch (Exception $ex) {
+        // There's no docblock metadata.
+      }
+
+      // If there's no group, but the file matches a group, use that group.
+      if ($group === null && isset($context['group'])) {
+        $group = $context['group'];
+      }
+
+      if ($group !== null) {
+        $atom->setProperty('group', $group);
+      }
+    }
+
+    return $atoms;
   }
 
   abstract protected function executeAtomize($file_name, $file_data);

src/applications/diviner/atomizer/DivinerPHPAtomizer.php

@@ -89,6 +89,8 @@ final class DivinerPHPAtomizer extends DivinerAtomizer {
             foreach ($attributes as $attribute) {
               $attr = strtolower($attribute->getConcreteString());
               switch ($attr) {
+                case 'final':
+                case 'abstract':
                 case 'static':
                   $matom->setProperty($attr, true);
                   break;

src/applications/diviner/controller/DivinerAtomController.php

@@ -81,7 +81,7 @@ final class DivinerAtomController extends DivinerController {
 
     $properties = id(new PhabricatorPropertyListView());
 
-    $group = $atom->getDocblockMetaValue('group');
+    $group = $atom->getProperty('group');
     if ($group) {
       $group_name = $book->getGroupName($group);
     } else {

src/applications/diviner/workflow/DivinerAtomizeWorkflow.php

@@ -63,7 +63,18 @@ final class DivinerAtomizeWorkflow extends DivinerWorkflow {
       $configure->setBook($this->getConfig('name'));
     }
 
+    $group_rules = array();
+    foreach ($this->getConfig('groups', array()) as $group => $spec) {
+      $include = (array)idx($spec, 'include', array());
+      foreach ($include as $pattern) {
+        $group_rules[$pattern] = $group;
+      }
+    }
+
     $all_atoms = array();
+    $context = array(
+      'group' => null,
+    );
     foreach ($files as $file) {
       $abs_path = Filesystem::resolvePath($file, $this->getConfig('root'));
       $data = Filesystem::readFile($abs_path);
@@ -75,7 +86,15 @@ final class DivinerAtomizeWorkflow extends DivinerWorkflow {
         $console->writeLog("Atomizing %s...\n", $file);
       }
 
-      $file_atoms = $file_atomizer->atomize($file, $data);
+      $context['group'] = null;
+      foreach ($group_rules as $rule => $group) {
+        if (preg_match($rule, $file)) {
+          $context['group'] = $group;
+          break;
+        }
+      }
+
+      $file_atoms = $file_atomizer->atomize($file, $data, $context);
       $all_atoms[] = $file_atoms;
 
       if (count($file_atoms) !== 1) {
@@ -83,7 +102,7 @@ final class DivinerAtomizeWorkflow extends DivinerWorkflow {
       }
       $file_atom = head($file_atoms);
 
-      $atoms = $atomizer->atomize($file, $data);
+      $atoms = $atomizer->atomize($file, $data, $context);
 
       foreach ($atoms as $atom) {
         if (!$atom->getParentHash()) {

src/applications/diviner/workflow/DivinerGenerateWorkflow.php

@@ -205,26 +205,11 @@ final class DivinerGenerateWorkflow extends DivinerWorkflow {
       '/\\.php$/' => 'DivinerPHPAtomizer',
     ));
 
-    foreach ($rules as $rule => $atomizer) {
-      if (@preg_match($rule, '') === false) {
-        throw new Exception(
-          "Rule '{$rule}' is not a valid regular expression!");
-      }
-    }
-
     return $rules;
   }
 
   private function getExclude() {
-    $exclude = $this->getConfig('exclude', array());
+    $exclude = (array)$this->getConfig('exclude', array());
-
-    foreach ($exclude as $rule) {
-      if (@preg_match($rule, '') === false) {
-        throw new Exception(
-          "Exclude rule '{$rule}' is not a valid regular expression!");
-      }
-    }
-
     return $exclude;
   }
 

src/applications/diviner/workflow/DivinerWorkflow.php

@@ -35,6 +35,19 @@ abstract class DivinerWorkflow extends PhutilArgumentWorkflow {
         "Book configuration '{$book_path}' is not in JSON format.");
     }
 
+    PhutilTypeSpec::checkMap(
+      $book,
+      array(
+        'name' => 'string',
+        'title' => 'optional string',
+        'short' => 'optional string',
+        'root' => 'optional string',
+        'uri.source' => 'optional string',
+        'rules' => 'optional map<regex, string>',
+        'exclude' => 'optional regex|list<regex>',
+        'groups' => 'optional map<string, map<string, wild>>',
+      ));
+
     // If the book specifies a "root", resolve it; otherwise, use the directory
     // the book configuration file lives in.
     $full_path = dirname(Filesystem::resolvePath($book_path));
@@ -43,13 +56,6 @@ abstract class DivinerWorkflow extends PhutilArgumentWorkflow {
     }
     $book['root'] = Filesystem::resolvePath($book['root'], $full_path);
 
-    // Make sure we have a valid book name.
-    if (!isset($book['name'])) {
-      throw new PhutilArgumentUsageException(
-        "Book configuration '{$book_path}' is missing required ".
-        "property 'name'.");
-    }
-
     if (!preg_match('/^[a-z][a-z-]*$/', $book['name'])) {
       $name = $book['name'];
       throw new PhutilArgumentUsageException(
@@ -57,6 +63,16 @@ abstract class DivinerWorkflow extends PhutilArgumentWorkflow {
         "must include only lowercase letters and hyphens.");
     }
 
+    foreach (idx($book, 'groups', array()) as $group) {
+      PhutilTypeSpec::checkmap(
+        $group,
+        array(
+          'name' => 'string',
+          'include' => 'optional regex|list<regex>',
+        ));
+
+    }
+
     $this->bookConfigPath = $book_path;
     $this->config = $book;
   }

src/docs/book/phabricator.book

@@ -17,5 +17,9 @@
     "(^src/docs/user/)"
   ],
   "groups" : {
+    "paste" : {
+      "name" : "Paste",
+      "include" : "(^src/applications/paste/)"
+    }
   }
 }