Continue construction of bin/celerity map

Summary: Ref T4222. Continues porting `scripts/celerity_mapper.php` functionality into `bin/celerity map`. This is pretty much a `1:1` port with no functional changes, but hopefully the code is a little better factored. Particularly, more responsibilities are pluggalbe through `CelerityResources` now.

Test Plan: Ran `bin/celerity map` and inexpected the `var_dump()` output, which appeared to make sense.

Reviewers: btrahan, hach-que

Reviewed By: hach-que

CC: aran

Maniphest Tasks: T4222

Differential Revision: https://secure.phabricator.com/D7865

src/infrastructure/celerity/CelerityResourceTransformer.php

@@ -7,6 +7,7 @@ final class CelerityResourceTransformer {
 
   private $minify;
   private $rawResourceMap;
+  private $rawURIMap;
   private $celerityMap;
   private $translateURICallback;
   private $currentPath;
@@ -31,6 +32,15 @@ final class CelerityResourceTransformer {
     return $this;
   }
 
+  public function setRawURIMap(array $raw_urimap) {
+    $this->rawURIMap = $raw_urimap;
+    return $this;
+  }
+
+  public function getRawURIMap() {
+    return $this->rawURIMap;
+  }
+
   /**
    * @phutil-external-symbol function jsShrink
    */
@@ -108,7 +118,11 @@ final class CelerityResourceTransformer {
   public function translateResourceURI(array $matches) {
     $uri = trim($matches[1], "'\" \r\t\n");
 
-    if ($this->rawResourceMap) {
+    if ($this->rawURIMap !== null) {
+      if (isset($this->rawURIMap[$uri])) {
+        $uri = $this->rawURIMap[$uri];
+      }
+    } else if ($this->rawResourceMap) {
       if (isset($this->rawResourceMap[$uri]['uri'])) {
         $uri = $this->rawResourceMap[$uri]['uri'];
       }

src/infrastructure/celerity/management/CelerityManagementMapWorkflow.php

@@ -16,12 +16,190 @@ final class CelerityManagementMapWorkflow
     $resources_map = CelerityResources::getAll();
 
     foreach ($resources_map as $name => $resources) {
-      // TODO: This does not do anything useful yet.
+      $this->rebuildResources($resources);
-      var_dump($resources->findBinaryResources());
-      var_dump($resources->findTextResources());
     }
 
     return 0;
   }
 
+  /**
+   * Rebuild the resource map for a resource source.
+   *
+   * @param CelerityResources Resource source to rebuild.
+   * @return void
+   */
+  private function rebuildResources(CelerityResources $resources) {
+    $binary_map = $this->rebuildBinaryResources($resources);
+
+    $xformer = id(new CelerityResourceTransformer())
+      ->setMinify(false)
+      ->setRawURIMap(ipull($binary_map, 'uri'));
+
+    $text_map = $this->rebuildTextResources($resources, $xformer);
+
+    $resource_graph = array();
+    $requires_map = array();
+    $provides_map = array();
+    foreach ($text_map as $name => $info) {
+      if (isset($info['provides'])) {
+        $provides_map[$info['provides']] = $info['hash'];
+
+        // We only need to check for cycles and add this to the requires map
+        // if it actually requires anything.
+        if (!empty($info['requires'])) {
+          $resource_graph[$info['provides']] = $info['requires'];
+          $requires_map[$info['hash']] = $info['requires'];
+        }
+      }
+    }
+
+    $this->detectGraphCycles($resource_graph);
+
+    $hash_map = ipull($binary_map, 'hash') + ipull($text_map, 'hash');
+
+
+    // TODO: Actually do things.
+
+    var_dump($provides_map);
+    var_dump($requires_map);
+    var_dump($hash_map);
+  }
+
+
+  /**
+   * Find binary resources (like PNG and SWF) and return information about
+   * them.
+   *
+   * @param CelerityResources Resource map to find binary resources for.
+   * @return map<string, map<string, string>> Resource information map.
+   */
+  private function rebuildBinaryResources(CelerityResources $resources) {
+    $binary_map = $resources->findBinaryResources();
+
+    $result_map = array();
+    foreach ($binary_map as $name => $data_hash) {
+      $hash = $resources->getCelerityHash($data_hash.$name);
+
+      $result_map[$name] = array(
+        'hash' => $hash,
+        'uri' => $resources->getResourceURI($hash, $name),
+      );
+    }
+
+    return $result_map;
+  }
+
+
+  /**
+   * Find text resources (like JS and CSS) and return information about them.
+   *
+   * @param CelerityResources Resource map to find text resources for.
+   * @param CelerityResourceTransformer Configured resource transformer.
+   * @return map<string, map<string, string>> Resource information map.
+   */
+  private function rebuildTextResources(
+    CelerityResources $resources,
+    CelerityResourceTransformer $xformer) {
+
+    $text_map = $resources->findTextResources();
+
+    $result_map = array();
+    foreach ($text_map as $name => $data_hash) {
+      $raw_data = $resources->getResourceData($name);
+      $xformed_data = $xformer->transformResource($name, $raw_data);
+
+      $data_hash = $resources->getCelerityHash($xformed_data);
+      $hash = $resources->getCelerityHash($data_hash.$name);
+
+      list($provides, $requires) = $this->getProvidesAndRequires(
+        $name,
+        $raw_data);
+
+      $result_map[$name] = array(
+        'hash' => $hash,
+      );
+
+      if ($provides !== null) {
+        $result_map[$name] += array(
+          'provides' => $provides,
+          'requires' => $requires,
+        );
+      }
+    }
+
+    return $result_map;
+  }
+
+
+  /**
+   * Parse the `@provides` and `@requires` symbols out of a text resource, like
+   * JS or CSS.
+   *
+   * @param string Resource name.
+   * @param string Resource data.
+   * @return pair<string|null, list<string>|nul> The `@provides` symbol and the
+   *    list of `@requires` symbols. If the resource is not part of the
+   *    dependency graph, both are null.
+   */
+  private function getProvidesAndRequires($name, $data) {
+    $parser = new PhutilDocblockParser();
+
+    $matches = array();
+    $ok = preg_match('@/[*][*].*?[*]/@s', $data, $matches);
+    if (!$ok) {
+      throw new Exception(
+        pht(
+          'Resource "%s" does not have a header doc comment. Encode '.
+          'dependency data in a header docblock.',
+          $name));
+    }
+
+    list($description, $metadata) = $parser->parse($matches[0]);
+
+    $provides = preg_split('/\s+/', trim(idx($metadata, 'provides')));
+    $requires = preg_split('/\s+/', trim(idx($metadata, 'requires')));
+    $provides = array_filter($provides);
+    $requires = array_filter($requires);
+
+    if (!$provides) {
+      // Tests and documentation-only JS is permitted to @provide no targets.
+      return array(null, null);
+    }
+
+    if (count($provides) > 1) {
+      throw new Exception(
+        pht(
+          'Resource "%s" must @provide at most one Celerity target.',
+          $name));
+    }
+
+    return array(head($provides), $requires);
+  }
+
+
+  /**
+   * Check for dependency cycles in the resource graph. Raises an exception if
+   * a cycle is detected.
+   *
+   * @param map<string, list<string>> Map of `@provides` symbols to their
+   *                                  `@requires` symbols.
+   * @return void
+   */
+  private function detectGraphCycles(array $nodes) {
+    $graph = id(new CelerityResourceGraph())
+      ->addNodes($nodes)
+      ->setResourceGraph($nodes)
+      ->loadGraph();
+
+    foreach ($nodes as $provides => $requires) {
+      $cycle = $graph->detectCycles($provides);
+      if ($cycle) {
+        throw new Exception(
+          pht(
+            'Cycle detected in resource graph: %s',
+            implode(' > ', $cycle)));
+      }
+    }
+  }
+
 }

src/infrastructure/celerity/resources/CelerityResources.php

@@ -7,11 +7,22 @@ abstract class CelerityResources {
 
   abstract public function getName();
   abstract public function getPathToMap();
+  abstract public function getResourceData($name);
   abstract public function findBinaryResources();
   abstract public function findTextResources();
 
-  public function getResourceHashKey() {
+  public function getCelerityHash($data) {
-    return PhabricatorEnv::getEnvConfig('celerity.resource-hash');
+    $tail = PhabricatorEnv::getEnvConfig('celerity.resource-hash');
+    $hash = PhabricatorHash::digest($data, $tail);
+    return substr($hash, 0, 8);
+  }
+
+  public function getResourceType($path) {
+    return CelerityResourceTransformer::getResourceType($path);
+  }
+
+  public function getResourceURI($hash, $name) {
+    return "/res/{$hash}/{$name}";
   }
 
   public static function getAll() {

src/infrastructure/celerity/resources/CelerityResourcesOnDisk.php

@@ -7,6 +7,10 @@ abstract class CelerityResourcesOnDisk extends CelerityResources {
 
   abstract public function getPathToResources();
 
+  public function getResourceData($name) {
+    return Filesystem::readFile($this->getPathToResources().'/'.$name);
+  }
+
   public function findBinaryResources() {
     return $this->findResourcesWithSuffixes($this->getBinaryFileSuffixes());
   }
@@ -47,7 +51,7 @@ abstract class CelerityResourcesOnDisk extends CelerityResources {
 
     $results = array();
     foreach ($raw_files as $path => $hash) {
-      $readable = '/'.Filesystem::readablePath($path, $root);
+      $readable = Filesystem::readablePath($path, $root);
       $results[$readable] = $hash;
     }