Provide a simpler library map rebuild script

Summary:
Modernize `phutil_mapper.php` to prepare for killing `__init__.php`.

The current mapper is module-oriented and complex. Instead, make the mapper file-oriented and simpler.

We build a one-to-one cache of file content to symbols (built with `phutil_symbols.php`) and then write a simpler map. See some discussion in D2561.

Also make the script less messy/bad in general. It may be useful to compare this to phutil_mapper.php.

(Additionally, we now write versions into the library map and cache.)

NOTE: Nothing can read this new map right now, of course.

Test Plan: Ran "phutil_rebuild_map.php src/" in phabricator/ with --quiet, --drop-caches, etc. Verified cache file, cache behavior, and generated map output.

Reviewers: vrana, nh, btrahan

Reviewed By: vrana

CC: aran

Maniphest Tasks: T1103

Differential Revision: https://secure.phabricator.com/D2562

scripts/phutil_rebuild_map.php

@@ -0,0 +1,478 @@
+#!/usr/bin/env php
+<?php
+
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+require_once dirname(__FILE__).'/__init_script__.php';
+
+$args = new PhutilArgumentParser($argv);
+$args->setTagline('rebuild the library map file');
+$args->setSynopsis(<<<EOHELP
+    **phutil_rebuild_map.php** [__options__] __root__
+        Rebuild the library map file for a libphutil library.
+
+EOHELP
+);
+
+$args->parseStandardArguments();
+$args->parse(
+  array(
+    array(
+      'name'      => 'quiet',
+      'help'      => 'Do not write status messages to stderr.',
+    ),
+    array(
+      'name'      => 'drop-cache',
+      'help'      => 'Drop the symbol cache and rebuild the entire map from '.
+                     'scratch.',
+    ),
+    array(
+      'name'      => 'root',
+      'wildcard'  => true,
+    )
+  ));
+
+$root = $args->getArg('root');
+if (count($root) !== 1) {
+  throw new Exception("Provide exactly one library root!");
+}
+$root = Filesystem::resolvePath(head($root));
+
+$builder = new PhutilLibraryMapBuilder($root);
+
+$builder->setQuiet($args->getArg('quiet'));
+
+if ($args->getArg('drop-cache')) {
+  $builder->dropSymbolCache();
+}
+
+$builder->buildMap();
+
+exit(0);
+
+
+
+
+/**
+ * Build maps of libphutil libraries. libphutil uses the library map to locate
+ * and load classes and functions in the library.
+ *
+ * @task map      Mapping libphutil Libraries
+ * @task path     Path Management
+ * @task symbol   Symbol Analysis and Caching
+ * @task source   Source Management
+ */
+final class PhutilLibraryMapBuilder {
+
+  private $root;
+  private $quiet;
+
+  const LIBRARY_MAP_VERSION_KEY   = '__library_version__';
+  const LIBRARY_MAP_VERSION       = 2;
+
+  const SYMBOL_CACHE_VERSION_KEY  = '__symbol_cache_version__';
+  const SYMBOL_CACHE_VERSION      = 2;
+
+
+/* -(  Mapping libphutil Libraries  )---------------------------------------- */
+
+
+  /**
+   * Create a new map builder for a library.
+   *
+   * @param string Path to the library root.
+   *
+   * @task map
+   */
+  public function __construct($root) {
+    $this->root = $root;
+  }
+
+
+  /**
+   * Control status output. Use --quiet to set this.
+   *
+   * @param   bool  If true, don't show status output.
+   * @return this
+   *
+   * @task map
+   */
+  public function setQuiet($quiet) {
+    $this->quiet = $quiet;
+    return $this;
+  }
+
+
+  /**
+   * Build or rebuild the library map.
+   *
+   * @return this
+   *
+   * @task map
+   */
+  public function buildMap() {
+
+    // Identify all the ".php" source files in the library.
+    $this->log("Finding source files...\n");
+    $source_map = $this->loadSourceFileMap();
+    $this->log("Found ".number_format(count($source_map))." files.\n");
+
+
+    // Load the symbol cache with existing parsed symbols. This allows us
+    // to remap libraries quickly by analyzing only changed files.
+    $this->log("Loading symbol cache...\n");
+    $symbol_cache = $this->loadSymbolCache();
+
+
+    // Build out the symbol analysis for all the files in the library. For
+    // each file, check if it's in cache. If we miss in the cache, do a fresh
+    // analysis.
+    $symbol_map = array();
+    $futures = array();
+    foreach ($source_map as $file => $hash) {
+      if (!empty($symbol_cache[$hash])) {
+        $symbol_map[$file] = $symbol_cache[$hash];
+        continue;
+      }
+      $futures[$file] = $this->buildSymbolAnalysisFuture($file);
+    }
+    $this->log("Found ".number_format(count($symbol_map))." files in cache.\n");
+
+
+    // Run the analyzer on any files which need analysis.
+    if ($futures) {
+      $this->log("Analyzing ".number_format(count($futures))." files");
+      foreach (Futures($futures)->limit(8) as $file => $future) {
+        $this->log(".");
+        $symbol_map[$file] = $future->resolveJSON();
+      }
+      $this->log("\nDone.\n");
+    }
+
+
+    // We're done building the cache, so write it out immediately. Note that
+    // we've only retained entries for files we found, so this implicitly cleans
+    // out old cache entries.
+    $this->writeSymbolCache($symbol_map, $source_map);
+
+    $this->log("Building library map...\n");
+    $library_map = $this->buildLibraryMap($symbol_map, $source_map);
+
+    $this->log("Writing map...\n");
+    $this->writeLibraryMap($library_map);
+
+    $this->log("Done.\n");
+
+    return $this;
+  }
+
+
+  /**
+   * Write a status message to the user, if not running in quiet mode.
+   *
+   * @param   string Message to write.
+   * @return  this
+   *
+   * @task map
+   */
+  private function log($message) {
+    if (!$this->quiet) {
+      @fwrite(STDERR, $message);
+    }
+    return $this;
+  }
+
+
+/* -(  Path Management  )---------------------------------------------------- */
+
+
+  /**
+   * Get the path to some file in the library.
+   *
+   * @param string  A library-relative path. If omitted, returns the library
+   *                root path.
+   * @return string An absolute path.
+   *
+   * @task path
+   */
+  private function getPath($path = '') {
+    return $this->root.'/'.$path;
+  }
+
+
+  /**
+   * Get the path to the symbol cache file.
+   *
+   * @return string Absolute path to symbol cache.
+   *
+   * @task path
+   */
+  private function getPathForSymbolCache() {
+    return $this->getPath('.phutil_module_cache');
+  }
+
+
+  /**
+   * Get the path to the map file.
+   *
+   * @return string Absolute path to the library map.
+   *
+   * @task path
+   */
+  private function getPathForLibraryMap() {
+    return $this->getPath('__phutil_library_map__.php');
+  }
+
+
+  /**
+   * Get the path to the library init file.
+   *
+   * @return string Absolute path to the library init file
+   *
+   * @task path
+   */
+  private function getPathForLibraryInit() {
+    return $this->getPath('__phutil_library_init__.php');
+  }
+
+
+/* -(  Symbol Analysis and Caching  )---------------------------------------- */
+
+
+  /**
+   * Load the library symbol cache, if it exists and is readable and valid.
+   *
+   * @return  dict  Map of content hashes to cache of output from
+   *                `phutil_symbols.php`.
+   *
+   * @task symbol
+   */
+  private function loadSymbolCache() {
+    $cache_file = $this->getPathForSymbolCache();
+
+    try {
+      $cache = Filesystem::readFile($cache_file);
+    } catch (Exception $ex) {
+      $cache = null;
+    }
+
+    $symbol_cache = array();
+    if ($cache) {
+      $symbol_cache = json_decode($cache, true);
+      if (!is_array($symbol_cache)) {
+        $symbol_cache = array();
+      }
+    }
+
+    $version = idx($symbol_cache, self::SYMBOL_CACHE_VERSION_KEY);
+    if ($version != self::SYMBOL_CACHE_VERSION) {
+      // Throw away caches from a different version of the library.
+      $symbol_cache = array();
+    }
+    unset($symbol_cache[self::SYMBOL_CACHE_VERSION_KEY]);
+
+    return $symbol_cache;
+  }
+
+
+  /**
+   * Write a symbol map to disk cache.
+   *
+   * @param dict  Symbol map of relative paths to symbols.
+   * @param dict  Source map (like @{method:loadSourceFileMap}).
+   * @return void
+   *
+   * @task symbol
+   */
+  private function writeSymbolCache(array $symbol_map, array $source_map) {
+    $cache_file = $this->getPathForSymbolCache();
+
+    $cache = array(
+      self::SYMBOL_CACHE_VERSION_KEY => self::SYMBOL_CACHE_VERSION,
+    );
+
+    foreach ($symbol_map as $file => $symbols) {
+      $cache[$source_map[$file]] = $symbols;
+    }
+
+    $json = json_encode($cache);
+    Filesystem::writeFile($cache_file, $json);
+  }
+
+
+  /**
+   * Drop the symbol cache, forcing a clean rebuild.
+   *
+   * @return this
+   *
+   * @task symbol
+   */
+  public function dropSymbolCache() {
+    $this->log("Dropping symbol cache...\n");
+    Filesystem::remove($this->getPathForSymbolCache());
+  }
+
+
+  /**
+   * Build a future which returns a `phutil_symbols.php` analysis of a source
+   * file.
+   *
+   * @param   string  Relative path to the source file to analyze.
+   * @return  Future  Analysis future.
+   *
+   * @task symbol
+   */
+  private function buildSymbolAnalysisFuture($file) {
+    $absolute_file = $this->getPath($file);
+    $bin = dirname(__FILE__).'/phutil_symbols.php';
+
+    return new ExecFuture('%s --ugly -- %s', $bin, $absolute_file);
+  }
+
+
+/* -(  Source Management  )-------------------------------------------------- */
+
+
+  /**
+   * Build a map of all source files in a library to hashes of their content.
+   * Returns an array like this:
+   *
+   *   array(
+   *     'src/parser/ExampleParser.php' => '60b725f10c9c85c70d97880dfe8191b3',
+   *     // ...
+   *   );
+   *
+   * @return  dict    Map of library-relative paths to content hashes.
+   * @task source
+   */
+  private function loadSourceFileMap() {
+    $root = $this->getPath();
+
+    $init = $this->getPathForLibraryInit();
+    if (!Filesystem::pathExists($init)) {
+      throw new Exception("Provided path '{$root}' is not a phutil library.");
+    }
+
+    $files = id(new FileFinder($root))
+      ->withType('f')
+      ->withSuffix('php')
+      ->excludePath('*/.*')
+      ->setGenerateChecksums(true)
+      ->find();
+
+    $map = array();
+    foreach ($files as $file => $hash) {
+      if (basename($file) == '__init__.php') {
+        // TODO: Remove this once we kill __init__.php. This just makes the
+        // script run faster until we do, so testing and development is less
+        // annoying.
+        continue;
+      }
+
+      $file = Filesystem::readablePath($file, $root);
+      $file = ltrim($file, '/');
+
+      if (dirname($file) == '.') {
+        // We don't permit normal source files at the root level, so just ignore
+        // them; they're special library files.
+        continue;
+      }
+
+      $map[$file] = $hash;
+    }
+
+    return $map;
+  }
+
+
+  /**
+   * Convert the symbol analysis of all the source files in the library into
+   * a library map.
+   *
+   * @param   dict  Symbol analysis of all source files.
+   * @return  dict  Library map.
+   * @task source
+   */
+  private function buildLibraryMap(array $symbol_map) {
+    $library_map = array();
+
+    // Detect duplicate symbols within the library.
+    foreach ($symbol_map as $file => $info) {
+      foreach ($info['have'] as $type => $symbols) {
+        foreach ($symbols as $symbol => $declaration) {
+          $lib_type = ($type == 'interface') ? 'class' : $type;
+          if (!empty($library_map[$lib_type][$symbol])) {
+            $prior = $library_map[$lib_type][$symbol];
+            throw new Exception(
+              "Definition of {$type} '{$symbol}' in file '{$file}' duplicates ".
+              "prior definition in file '{$prior}'. You can not declare the ".
+              "same symbol twice.");
+          }
+          $library_map[$lib_type][$symbol] = $file;
+        }
+      }
+    }
+
+    // Sort the map so it is relatively stable across changes.
+    foreach ($library_map as $lib_type => $symbols) {
+      ksort($symbols);
+      $library_map[$lib_type] = $symbols;
+    }
+    ksort($library_map);
+
+    return $library_map;
+  }
+
+
+  /**
+   * Write a finalized library map.
+   *
+   * @param  dict Library map structure to write.
+   * @return void
+   *
+   * @task source
+   */
+  private function writeLibraryMap(array $library_map) {
+    $map_file = $this->getPathForLibraryMap();
+    $version = self::LIBRARY_MAP_VERSION;
+
+    $library_map = array(
+      self::LIBRARY_MAP_VERSION_KEY => $version,
+    ) + $library_map;
+
+    $library_map = var_export($library_map, $return_string = true);
+    $library_map = preg_replace('/\s+$/m', '', $library_map);
+    $library_map = preg_replace('/array \(/', 'array(', $library_map);
+    $at = '@';
+
+    $source_file = <<<EOPHP
+<?php
+
+/**
+ * This file is automatically generated. Use 'arc liberate' to rebuild it.
+ * {$at}generated
+ * {$at}phutil-library-version {$version}
+ */
+
+phutil_register_library_map({$library_map});
+
+EOPHP;
+
+    Filesystem::writeFile($map_file, $source_file);
+  }
+
+}