mirror of
https://we.phorge.it/source/arcanist.git
synced 2024-11-22 06:42:41 +01:00
Provide a simpler library map rebuild script
Summary: Modernize `phutil_mapper.php` to prepare for killing `__init__.php`. The current mapper is module-oriented and complex. Instead, make the mapper file-oriented and simpler. We build a one-to-one cache of file content to symbols (built with `phutil_symbols.php`) and then write a simpler map. See some discussion in D2561. Also make the script less messy/bad in general. It may be useful to compare this to phutil_mapper.php. (Additionally, we now write versions into the library map and cache.) NOTE: Nothing can read this new map right now, of course. Test Plan: Ran "phutil_rebuild_map.php src/" in phabricator/ with --quiet, --drop-caches, etc. Verified cache file, cache behavior, and generated map output. Reviewers: vrana, nh, btrahan Reviewed By: vrana CC: aran Maniphest Tasks: T1103 Differential Revision: https://secure.phabricator.com/D2562
This commit is contained in:
epriestley
parent
c51590dc1e
commit
e9a6cd26fc
1 changed files with 478 additions and 0 deletions
478
scripts/phutil_rebuild_map.php
Executable file
478
scripts/phutil_rebuild_map.php
Executable file
|
|
@ -0,0 +1,478 @@
|
|||
#!/usr/bin/env php
|
||||
<?php
|
||||
|
||||
/*
|
||||
* Copyright 2012 Facebook, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
require_once dirname(__FILE__).'/__init_script__.php';
|
||||
|
||||
$args = new PhutilArgumentParser($argv);
|
||||
$args->setTagline('rebuild the library map file');
|
||||
$args->setSynopsis(<<<EOHELP
|
||||
**phutil_rebuild_map.php** [__options__] __root__
|
||||
Rebuild the library map file for a libphutil library.
|
||||
|
||||
EOHELP
|
||||
);
|
||||
|
||||
$args->parseStandardArguments();
|
||||
$args->parse(
|
||||
array(
|
||||
array(
|
||||
'name' => 'quiet',
|
||||
'help' => 'Do not write status messages to stderr.',
|
||||
),
|
||||
array(
|
||||
'name' => 'drop-cache',
|
||||
'help' => 'Drop the symbol cache and rebuild the entire map from '.
|
||||
'scratch.',
|
||||
),
|
||||
array(
|
||||
'name' => 'root',
|
||||
'wildcard' => true,
|
||||
)
|
||||
));
|
||||
|
||||
$root = $args->getArg('root');
|
||||
if (count($root) !== 1) {
|
||||
throw new Exception("Provide exactly one library root!");
|
||||
}
|
||||
$root = Filesystem::resolvePath(head($root));
|
||||
|
||||
$builder = new PhutilLibraryMapBuilder($root);
|
||||
|
||||
$builder->setQuiet($args->getArg('quiet'));
|
||||
|
||||
if ($args->getArg('drop-cache')) {
|
||||
$builder->dropSymbolCache();
|
||||
}
|
||||
|
||||
$builder->buildMap();
|
||||
|
||||
exit(0);
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Build maps of libphutil libraries. libphutil uses the library map to locate
|
||||
* and load classes and functions in the library.
|
||||
*
|
||||
* @task map Mapping libphutil Libraries
|
||||
* @task path Path Management
|
||||
* @task symbol Symbol Analysis and Caching
|
||||
* @task source Source Management
|
||||
*/
|
||||
final class PhutilLibraryMapBuilder {
|
||||
|
||||
private $root;
|
||||
private $quiet;
|
||||
|
||||
const LIBRARY_MAP_VERSION_KEY = '__library_version__';
|
||||
const LIBRARY_MAP_VERSION = 2;
|
||||
|
||||
const SYMBOL_CACHE_VERSION_KEY = '__symbol_cache_version__';
|
||||
const SYMBOL_CACHE_VERSION = 2;
|
||||
|
||||
|
||||
/* -( Mapping libphutil Libraries )---------------------------------------- */
|
||||
|
||||
|
||||
/**
|
||||
* Create a new map builder for a library.
|
||||
*
|
||||
* @param string Path to the library root.
|
||||
*
|
||||
* @task map
|
||||
*/
|
||||
public function __construct($root) {
|
||||
$this->root = $root;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Control status output. Use --quiet to set this.
|
||||
*
|
||||
* @param bool If true, don't show status output.
|
||||
* @return this
|
||||
*
|
||||
* @task map
|
||||
*/
|
||||
public function setQuiet($quiet) {
|
||||
$this->quiet = $quiet;
|
||||
return $this;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Build or rebuild the library map.
|
||||
*
|
||||
* @return this
|
||||
*
|
||||
* @task map
|
||||
*/
|
||||
public function buildMap() {
|
||||
|
||||
// Identify all the ".php" source files in the library.
|
||||
$this->log("Finding source files...\n");
|
||||
$source_map = $this->loadSourceFileMap();
|
||||
$this->log("Found ".number_format(count($source_map))." files.\n");
|
||||
|
||||
|
||||
// Load the symbol cache with existing parsed symbols. This allows us
|
||||
// to remap libraries quickly by analyzing only changed files.
|
||||
$this->log("Loading symbol cache...\n");
|
||||
$symbol_cache = $this->loadSymbolCache();
|
||||
|
||||
|
||||
// Build out the symbol analysis for all the files in the library. For
|
||||
// each file, check if it's in cache. If we miss in the cache, do a fresh
|
||||
// analysis.
|
||||
$symbol_map = array();
|
||||
$futures = array();
|
||||
foreach ($source_map as $file => $hash) {
|
||||
if (!empty($symbol_cache[$hash])) {
|
||||
$symbol_map[$file] = $symbol_cache[$hash];
|
||||
continue;
|
||||
}
|
||||
$futures[$file] = $this->buildSymbolAnalysisFuture($file);
|
||||
}
|
||||
$this->log("Found ".number_format(count($symbol_map))." files in cache.\n");
|
||||
|
||||
|
||||
// Run the analyzer on any files which need analysis.
|
||||
if ($futures) {
|
||||
$this->log("Analyzing ".number_format(count($futures))." files");
|
||||
foreach (Futures($futures)->limit(8) as $file => $future) {
|
||||
$this->log(".");
|
||||
$symbol_map[$file] = $future->resolveJSON();
|
||||
}
|
||||
$this->log("\nDone.\n");
|
||||
}
|
||||
|
||||
|
||||
// We're done building the cache, so write it out immediately. Note that
|
||||
// we've only retained entries for files we found, so this implicitly cleans
|
||||
// out old cache entries.
|
||||
$this->writeSymbolCache($symbol_map, $source_map);
|
||||
|
||||
$this->log("Building library map...\n");
|
||||
$library_map = $this->buildLibraryMap($symbol_map, $source_map);
|
||||
|
||||
$this->log("Writing map...\n");
|
||||
$this->writeLibraryMap($library_map);
|
||||
|
||||
$this->log("Done.\n");
|
||||
|
||||
return $this;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Write a status message to the user, if not running in quiet mode.
|
||||
*
|
||||
* @param string Message to write.
|
||||
* @return this
|
||||
*
|
||||
* @task map
|
||||
*/
|
||||
private function log($message) {
|
||||
if (!$this->quiet) {
|
||||
@fwrite(STDERR, $message);
|
||||
}
|
||||
return $this;
|
||||
}
|
||||
|
||||
|
||||
/* -( Path Management )---------------------------------------------------- */
|
||||
|
||||
|
||||
/**
|
||||
* Get the path to some file in the library.
|
||||
*
|
||||
* @param string A library-relative path. If omitted, returns the library
|
||||
* root path.
|
||||
* @return string An absolute path.
|
||||
*
|
||||
* @task path
|
||||
*/
|
||||
private function getPath($path = '') {
|
||||
return $this->root.'/'.$path;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get the path to the symbol cache file.
|
||||
*
|
||||
* @return string Absolute path to symbol cache.
|
||||
*
|
||||
* @task path
|
||||
*/
|
||||
private function getPathForSymbolCache() {
|
||||
return $this->getPath('.phutil_module_cache');
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get the path to the map file.
|
||||
*
|
||||
* @return string Absolute path to the library map.
|
||||
*
|
||||
* @task path
|
||||
*/
|
||||
private function getPathForLibraryMap() {
|
||||
return $this->getPath('__phutil_library_map__.php');
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get the path to the library init file.
|
||||
*
|
||||
* @return string Absolute path to the library init file
|
||||
*
|
||||
* @task path
|
||||
*/
|
||||
private function getPathForLibraryInit() {
|
||||
return $this->getPath('__phutil_library_init__.php');
|
||||
}
|
||||
|
||||
|
||||
/* -( Symbol Analysis and Caching )---------------------------------------- */
|
||||
|
||||
|
||||
/**
|
||||
* Load the library symbol cache, if it exists and is readable and valid.
|
||||
*
|
||||
* @return dict Map of content hashes to cache of output from
|
||||
* `phutil_symbols.php`.
|
||||
*
|
||||
* @task symbol
|
||||
*/
|
||||
private function loadSymbolCache() {
|
||||
$cache_file = $this->getPathForSymbolCache();
|
||||
|
||||
try {
|
||||
$cache = Filesystem::readFile($cache_file);
|
||||
} catch (Exception $ex) {
|
||||
$cache = null;
|
||||
}
|
||||
|
||||
$symbol_cache = array();
|
||||
if ($cache) {
|
||||
$symbol_cache = json_decode($cache, true);
|
||||
if (!is_array($symbol_cache)) {
|
||||
$symbol_cache = array();
|
||||
}
|
||||
}
|
||||
|
||||
$version = idx($symbol_cache, self::SYMBOL_CACHE_VERSION_KEY);
|
||||
if ($version != self::SYMBOL_CACHE_VERSION) {
|
||||
// Throw away caches from a different version of the library.
|
||||
$symbol_cache = array();
|
||||
}
|
||||
unset($symbol_cache[self::SYMBOL_CACHE_VERSION_KEY]);
|
||||
|
||||
return $symbol_cache;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Write a symbol map to disk cache.
|
||||
*
|
||||
* @param dict Symbol map of relative paths to symbols.
|
||||
* @param dict Source map (like @{method:loadSourceFileMap}).
|
||||
* @return void
|
||||
*
|
||||
* @task symbol
|
||||
*/
|
||||
private function writeSymbolCache(array $symbol_map, array $source_map) {
|
||||
$cache_file = $this->getPathForSymbolCache();
|
||||
|
||||
$cache = array(
|
||||
self::SYMBOL_CACHE_VERSION_KEY => self::SYMBOL_CACHE_VERSION,
|
||||
);
|
||||
|
||||
foreach ($symbol_map as $file => $symbols) {
|
||||
$cache[$source_map[$file]] = $symbols;
|
||||
}
|
||||
|
||||
$json = json_encode($cache);
|
||||
Filesystem::writeFile($cache_file, $json);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Drop the symbol cache, forcing a clean rebuild.
|
||||
*
|
||||
* @return this
|
||||
*
|
||||
* @task symbol
|
||||
*/
|
||||
public function dropSymbolCache() {
|
||||
$this->log("Dropping symbol cache...\n");
|
||||
Filesystem::remove($this->getPathForSymbolCache());
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Build a future which returns a `phutil_symbols.php` analysis of a source
|
||||
* file.
|
||||
*
|
||||
* @param string Relative path to the source file to analyze.
|
||||
* @return Future Analysis future.
|
||||
*
|
||||
* @task symbol
|
||||
*/
|
||||
private function buildSymbolAnalysisFuture($file) {
|
||||
$absolute_file = $this->getPath($file);
|
||||
$bin = dirname(__FILE__).'/phutil_symbols.php';
|
||||
|
||||
return new ExecFuture('%s --ugly -- %s', $bin, $absolute_file);
|
||||
}
|
||||
|
||||
|
||||
/* -( Source Management )-------------------------------------------------- */
|
||||
|
||||
|
||||
/**
|
||||
* Build a map of all source files in a library to hashes of their content.
|
||||
* Returns an array like this:
|
||||
*
|
||||
* array(
|
||||
* 'src/parser/ExampleParser.php' => '60b725f10c9c85c70d97880dfe8191b3',
|
||||
* // ...
|
||||
* );
|
||||
*
|
||||
* @return dict Map of library-relative paths to content hashes.
|
||||
* @task source
|
||||
*/
|
||||
private function loadSourceFileMap() {
|
||||
$root = $this->getPath();
|
||||
|
||||
$init = $this->getPathForLibraryInit();
|
||||
if (!Filesystem::pathExists($init)) {
|
||||
throw new Exception("Provided path '{$root}' is not a phutil library.");
|
||||
}
|
||||
|
||||
$files = id(new FileFinder($root))
|
||||
->withType('f')
|
||||
->withSuffix('php')
|
||||
->excludePath('*/.*')
|
||||
->setGenerateChecksums(true)
|
||||
->find();
|
||||
|
||||
$map = array();
|
||||
foreach ($files as $file => $hash) {
|
||||
if (basename($file) == '__init__.php') {
|
||||
// TODO: Remove this once we kill __init__.php. This just makes the
|
||||
// script run faster until we do, so testing and development is less
|
||||
// annoying.
|
||||
continue;
|
||||
}
|
||||
|
||||
$file = Filesystem::readablePath($file, $root);
|
||||
$file = ltrim($file, '/');
|
||||
|
||||
if (dirname($file) == '.') {
|
||||
// We don't permit normal source files at the root level, so just ignore
|
||||
// them; they're special library files.
|
||||
continue;
|
||||
}
|
||||
|
||||
$map[$file] = $hash;
|
||||
}
|
||||
|
||||
return $map;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Convert the symbol analysis of all the source files in the library into
|
||||
* a library map.
|
||||
*
|
||||
* @param dict Symbol analysis of all source files.
|
||||
* @return dict Library map.
|
||||
* @task source
|
||||
*/
|
||||
private function buildLibraryMap(array $symbol_map) {
|
||||
$library_map = array();
|
||||
|
||||
// Detect duplicate symbols within the library.
|
||||
foreach ($symbol_map as $file => $info) {
|
||||
foreach ($info['have'] as $type => $symbols) {
|
||||
foreach ($symbols as $symbol => $declaration) {
|
||||
$lib_type = ($type == 'interface') ? 'class' : $type;
|
||||
if (!empty($library_map[$lib_type][$symbol])) {
|
||||
$prior = $library_map[$lib_type][$symbol];
|
||||
throw new Exception(
|
||||
"Definition of {$type} '{$symbol}' in file '{$file}' duplicates ".
|
||||
"prior definition in file '{$prior}'. You can not declare the ".
|
||||
"same symbol twice.");
|
||||
}
|
||||
$library_map[$lib_type][$symbol] = $file;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort the map so it is relatively stable across changes.
|
||||
foreach ($library_map as $lib_type => $symbols) {
|
||||
ksort($symbols);
|
||||
$library_map[$lib_type] = $symbols;
|
||||
}
|
||||
ksort($library_map);
|
||||
|
||||
return $library_map;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Write a finalized library map.
|
||||
*
|
||||
* @param dict Library map structure to write.
|
||||
* @return void
|
||||
*
|
||||
* @task source
|
||||
*/
|
||||
private function writeLibraryMap(array $library_map) {
|
||||
$map_file = $this->getPathForLibraryMap();
|
||||
$version = self::LIBRARY_MAP_VERSION;
|
||||
|
||||
$library_map = array(
|
||||
self::LIBRARY_MAP_VERSION_KEY => $version,
|
||||
) + $library_map;
|
||||
|
||||
$library_map = var_export($library_map, $return_string = true);
|
||||
$library_map = preg_replace('/\s+$/m', '', $library_map);
|
||||
$library_map = preg_replace('/array \(/', 'array(', $library_map);
|
||||
$at = '@';
|
||||
|
||||
$source_file = <<<EOPHP
|
||||
<?php
|
||||
|
||||
/**
|
||||
* This file is automatically generated. Use 'arc liberate' to rebuild it.
|
||||
* {$at}generated
|
||||
* {$at}phutil-library-version {$version}
|
||||
*/
|
||||
|
||||
phutil_register_library_map({$library_map});
|
||||
|
||||
EOPHP;
|
||||
|
||||
Filesystem::writeFile($map_file, $source_file);
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in a new issue