phorge-arcanist/scripts/phutil_rebuild_map.php

#!/usr/bin/env php
<?php

/*
 * Copyright 2012 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

require_once dirname(__FILE__).'/__init_script__.php';

$args = new PhutilArgumentParser($argv);
$args->setTagline('rebuild the library map file');
$args->setSynopsis(<<<EOHELP
    **phutil_rebuild_map.php** [__options__] __root__
        Rebuild the library map file for a libphutil library.

EOHELP
);

$args->parseStandardArguments();
$args->parse(
  array(
    array(
      'name'      => 'quiet',
      'help'      => 'Do not write status messages to stderr.',
    ),
    array(
      'name'      => 'drop-cache',
      'help'      => 'Drop the symbol cache and rebuild the entire map from '.
                     'scratch.',
    ),
    array(
      'name'      => 'limit',
      'param'     => 'N',
      'default'   => 8,
      'help'      => 'Controls the number of symbol mapper subprocesses run '.
                     'at once. Defaults to 8.',
    ),
    array(
      'name'      => 'show',
      'help'      => 'Print symbol map to stdout instead of writing it to the '.
                     'map file.',
    ),
    array(
      'name'      => 'ugly',
      'help'      => 'Use faster but less readable serialization for --show.',
    ),
    array(
      'name'      => 'root',
      'wildcard'  => true,
    )
  ));

$root = $args->getArg('root');
if (count($root) !== 1) {
  throw new Exception("Provide exactly one library root!");
}
$root = Filesystem::resolvePath(head($root));

$builder = new PhutilLibraryMapBuilder($root);

$builder->setQuiet($args->getArg('quiet'));
$builder->setSubprocessLimit($args->getArg('limit'));

if ($args->getArg('drop-cache')) {
  $builder->dropSymbolCache();
}

if ($args->getArg('show')) {
  $builder->setShowMap(true);
  $builder->setUgly($args->getArg('ugly'));
}

$builder->buildMap();

exit(0);




/**
 * Build maps of libphutil libraries. libphutil uses the library map to locate
 * and load classes and functions in the library.
 *
 * @task map      Mapping libphutil Libraries
 * @task path     Path Management
 * @task symbol   Symbol Analysis and Caching
 * @task source   Source Management
 */
final class PhutilLibraryMapBuilder {

  private $root;
  private $quiet;
  private $subprocessLimit = 8;
  private $ugly;
  private $showMap;

  const LIBRARY_MAP_VERSION_KEY   = '__library_version__';
  const LIBRARY_MAP_VERSION       = 2;

  const SYMBOL_CACHE_VERSION_KEY  = '__symbol_cache_version__';
  const SYMBOL_CACHE_VERSION      = 2;


/* -(  Mapping libphutil Libraries  )---------------------------------------- */


  /**
   * Create a new map builder for a library.
   *
   * @param string Path to the library root.
   *
   * @task map
   */
  public function __construct($root) {
    $this->root = $root;
  }


  /**
   * Control status output. Use --quiet to set this.
   *
   * @param   bool  If true, don't show status output.
   * @return  this
   *
   * @task map
   */
  public function setQuiet($quiet) {
    $this->quiet = $quiet;
    return $this;
  }


  /**
   * Control subprocess parallelism limit. Use --limit to set this.
   *
   * @param   int   Maximum number of subprocesses to run in parallel.
   * @return  this
   *
   * @task map
   */
  public function setSubprocessLimit($limit) {
    $this->subprocessLimit = $limit;
    return $this;
  }


  /**
   * Control whether the ugly (but fast) or pretty (but slower) JSON formatter
   * is used.
   *
   * @param   bool  If true, use the fastest formatter.
   * @return  this
   *
   * @task map
   */
  public function setUgly($ugly) {
    $this->ugly = $ugly;
    return $this;
  }


  /**
   * Control whether the map should be rebuilt, or just shown (printed to
   * stdout in JSON).
   *
   * @param bool  If true, show map instead of updating.
   * @return this
   *
   * @task map
   */
  public function setShowMap($show_map) {
    $this->showMap = $show_map;
    return $this;
  }


  /**
   * Build or rebuild the library map.
   *
   * @return this
   *
   * @task map
   */
  public function buildMap() {

    // Identify all the ".php" source files in the library.
    $this->log("Finding source files...\n");
    $source_map = $this->loadSourceFileMap();
    $this->log("Found ".number_format(count($source_map))." files.\n");


    // Load the symbol cache with existing parsed symbols. This allows us
    // to remap libraries quickly by analyzing only changed files.
    $this->log("Loading symbol cache...\n");
    $symbol_cache = $this->loadSymbolCache();


    // Build out the symbol analysis for all the files in the library. For
    // each file, check if it's in cache. If we miss in the cache, do a fresh
    // analysis.
    $symbol_map = array();
    $futures = array();
    foreach ($source_map as $file => $hash) {
      if (!empty($symbol_cache[$hash])) {
        $symbol_map[$file] = $symbol_cache[$hash];
        continue;
      }
      $futures[$file] = $this->buildSymbolAnalysisFuture($file);
    }
    $this->log("Found ".number_format(count($symbol_map))." files in cache.\n");


    // Run the analyzer on any files which need analysis.
    if ($futures) {
      $limit = $this->subprocessLimit;
      $count = number_format(count($futures));

      $this->log("Analyzing {$count} files with {$limit} subprocesses...\n");

      foreach (Futures($futures)->limit($limit) as $file => $future) {
        $result = $future->resolveJSON();
        if (empty($result['error'])) {
          $symbol_map[$file] = $result;
        } else {
          echo phutil_console_format(
            "\n**SYNTAX ERROR!**\nFile: %s\nLine: %d\n\n%s\n",
            Filesystem::readablePath($result['file']),
            $result['line'],
            $result['error']);
          exit(1);
        }
        $this->log(".");
      }
      $this->log("\nDone.\n");
    }


    // We're done building the cache, so write it out immediately. Note that
    // we've only retained entries for files we found, so this implicitly cleans
    // out old cache entries.
    $this->writeSymbolCache($symbol_map, $source_map);


    // Our map is up to date, so either show it on stdout or write it to disk.

    if ($this->showMap) {
      $this->log("Showing map...\n");

      if ($this->ugly) {
        echo json_encode($symbol_map);
      } else {
        $json = new PhutilJSON();
        echo $json->encodeFormatted($symbol_map);
      }
    } else {
      $this->log("Building library map...\n");
      $library_map = $this->buildLibraryMap($symbol_map, $source_map);

      $this->log("Writing map...\n");
      $this->writeLibraryMap($library_map);
    }

    $this->log("Done.\n");

    return $this;
  }


  /**
   * Write a status message to the user, if not running in quiet mode.
   *
   * @param   string Message to write.
   * @return  this
   *
   * @task map
   */
  private function log($message) {
    if (!$this->quiet) {
      @fwrite(STDERR, $message);
    }
    return $this;
  }


/* -(  Path Management  )---------------------------------------------------- */


  /**
   * Get the path to some file in the library.
   *
   * @param string  A library-relative path. If omitted, returns the library
   *                root path.
   * @return string An absolute path.
   *
   * @task path
   */
  private function getPath($path = '') {
    return $this->root.'/'.$path;
  }


  /**
   * Get the path to the symbol cache file.
   *
   * @return string Absolute path to symbol cache.
   *
   * @task path
   */
  private function getPathForSymbolCache() {
    return $this->getPath('.phutil_module_cache');
  }


  /**
   * Get the path to the map file.
   *
   * @return string Absolute path to the library map.
   *
   * @task path
   */
  private function getPathForLibraryMap() {
    return $this->getPath('__phutil_library_map__.php');
  }


  /**
   * Get the path to the library init file.
   *
   * @return string Absolute path to the library init file
   *
   * @task path
   */
  private function getPathForLibraryInit() {
    return $this->getPath('__phutil_library_init__.php');
  }


/* -(  Symbol Analysis and Caching  )---------------------------------------- */


  /**
   * Load the library symbol cache, if it exists and is readable and valid.
   *
   * @return  dict  Map of content hashes to cache of output from
   *                `phutil_symbols.php`.
   *
   * @task symbol
   */
  private function loadSymbolCache() {
    $cache_file = $this->getPathForSymbolCache();

    try {
      $cache = Filesystem::readFile($cache_file);
    } catch (Exception $ex) {
      $cache = null;
    }

    $symbol_cache = array();
    if ($cache) {
      $symbol_cache = json_decode($cache, true);
      if (!is_array($symbol_cache)) {
        $symbol_cache = array();
      }
    }

    $version = idx($symbol_cache, self::SYMBOL_CACHE_VERSION_KEY);
    if ($version != self::SYMBOL_CACHE_VERSION) {
      // Throw away caches from a different version of the library.
      $symbol_cache = array();
    }
    unset($symbol_cache[self::SYMBOL_CACHE_VERSION_KEY]);

    return $symbol_cache;
  }


  /**
   * Write a symbol map to disk cache.
   *
   * @param dict  Symbol map of relative paths to symbols.
   * @param dict  Source map (like @{method:loadSourceFileMap}).
   * @return void
   *
   * @task symbol
   */
  private function writeSymbolCache(array $symbol_map, array $source_map) {
    $cache_file = $this->getPathForSymbolCache();

    $cache = array(
      self::SYMBOL_CACHE_VERSION_KEY => self::SYMBOL_CACHE_VERSION,
    );

    foreach ($symbol_map as $file => $symbols) {
      $cache[$source_map[$file]] = $symbols;
    }

    $json = json_encode($cache);
    Filesystem::writeFile($cache_file, $json);
  }


  /**
   * Drop the symbol cache, forcing a clean rebuild.
   *
   * @return this
   *
   * @task symbol
   */
  public function dropSymbolCache() {
    $this->log("Dropping symbol cache...\n");
    Filesystem::remove($this->getPathForSymbolCache());
  }


  /**
   * Build a future which returns a `phutil_symbols.php` analysis of a source
   * file.
   *
   * @param   string  Relative path to the source file to analyze.
   * @return  Future  Analysis future.
   *
   * @task symbol
   */
  private function buildSymbolAnalysisFuture($file) {
    $absolute_file = $this->getPath($file);
    $bin = dirname(__FILE__).'/phutil_symbols.php';

    return new ExecFuture('%s --ugly -- %s', $bin, $absolute_file);
  }


/* -(  Source Management  )-------------------------------------------------- */


  /**
   * Build a map of all source files in a library to hashes of their content.
   * Returns an array like this:
   *
   *   array(
   *     'src/parser/ExampleParser.php' => '60b725f10c9c85c70d97880dfe8191b3',
   *     // ...
   *   );
   *
   * @return  dict    Map of library-relative paths to content hashes.
   * @task source
   */
  private function loadSourceFileMap() {
    $root = $this->getPath();

    $init = $this->getPathForLibraryInit();
    if (!Filesystem::pathExists($init)) {
      throw new Exception("Provided path '{$root}' is not a phutil library.");
    }

    $files = id(new FileFinder($root))
      ->withType('f')
      ->withSuffix('php')
      ->excludePath('*/.*')
      ->setGenerateChecksums(true)
      ->find();

    $map = array();
    foreach ($files as $file => $hash) {
      if (basename($file) == '__init__.php') {
        // TODO: Remove this once we kill __init__.php. This just makes the
        // script run faster until we do, so testing and development is less
        // annoying.
        continue;
      }

      $file = Filesystem::readablePath($file, $root);
      $file = ltrim($file, '/');

      if (dirname($file) == '.') {
        // We don't permit normal source files at the root level, so just ignore
        // them; they're special library files.
        continue;
      }

      $map[$file] = $hash;
    }

    return $map;
  }


  /**
   * Convert the symbol analysis of all the source files in the library into
   * a library map.
   *
   * @param   dict  Symbol analysis of all source files.
   * @return  dict  Library map.
   * @task source
   */
  private function buildLibraryMap(array $symbol_map) {
    $library_map = array(
      'class'     => array(),
      'function'  => array(),
      'xmap'      => array(),
    );

    // Detect duplicate symbols within the library.
    foreach ($symbol_map as $file => $info) {
      foreach ($info['have'] as $type => $symbols) {
        foreach ($symbols as $symbol => $declaration) {
          $lib_type = ($type == 'interface') ? 'class' : $type;
          if (!empty($library_map[$lib_type][$symbol])) {
            $prior = $library_map[$lib_type][$symbol];
            throw new Exception(
              "Definition of {$type} '{$symbol}' in file '{$file}' duplicates ".
              "prior definition in file '{$prior}'. You can not declare the ".
              "same symbol twice.");
          }
          $library_map[$lib_type][$symbol] = $file;
        }
      }
      $library_map['xmap'] += $info['xmap'];
    }

    // Simplify the common case (one parent) to make the file a little easier
    // to deal with.
    foreach ($library_map['xmap'] as $class => $extends) {
      if (count($extends) == 1) {
        $library_map['xmap'][$class] = reset($extends);
      }
    }

    // Sort the map so it is relatively stable across changes.
    foreach ($library_map as $key => $symbols) {
      ksort($symbols);
      $library_map[$key] = $symbols;
    }
    ksort($library_map);

    return $library_map;
  }


  /**
   * Write a finalized library map.
   *
   * @param  dict Library map structure to write.
   * @return void
   *
   * @task source
   */
  private function writeLibraryMap(array $library_map) {
    $map_file = $this->getPathForLibraryMap();
    $version = self::LIBRARY_MAP_VERSION;

    $library_map = array(
      self::LIBRARY_MAP_VERSION_KEY => $version,
    ) + $library_map;

    $library_map = var_export($library_map, $return_string = true);
    $library_map = preg_replace('/\s+$/m', '', $library_map);
    $library_map = preg_replace('/array \(/', 'array(', $library_map);
    $at = '@';

    $source_file = <<<EOPHP
<?php

/**
 * This file is automatically generated. Use 'arc liberate' to rebuild it.
 * {$at}generated
 * {$at}phutil-library-version {$version}
 */

phutil_register_library_map({$library_map});

EOPHP;

    Filesystem::writeFile($map_file, $source_file);
  }

}