From c7e12c6a85c781d0ff3b7961bddec55d068722e5 Mon Sep 17 00:00:00 2001 From: epriestley Date: Sun, 17 Feb 2013 15:40:00 -0800 Subject: [PATCH] Add a publish cache for the Diviner static publisher Summary: Keep track of what we've written to disk, and regenerate only new documents. Test Plan: Changed a small number of files, saw that number of files get regenerated. Ran with "--clean" and saw everything regenerate. Reviewers: btrahan, vrana, chad Reviewed By: chad CC: aran Maniphest Tasks: T988 Differential Revision: https://secure.phabricator.com/D4897 --- src/__phutil_library_map__.php | 4 + .../diviner/cache/DivinerAtomCache.php | 35 +-------- .../diviner/cache/DivinerDiskCache.php | 42 +++++++++++ .../diviner/cache/DivinerPublishCache.php | 45 +++++++++++ .../diviner/publisher/DivinerPublisher.php | 5 +- .../publisher/DivinerStaticPublisher.php | 75 +++++++++++++++++-- 6 files changed, 167 insertions(+), 39 deletions(-) create mode 100644 src/applications/diviner/cache/DivinerDiskCache.php create mode 100644 src/applications/diviner/cache/DivinerPublishCache.php diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 8b5209eab1..2e5d1afcc9 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -468,9 +468,11 @@ phutil_register_library_map(array( 'DivinerAtomizeWorkflow' => 'applications/diviner/workflow/DivinerAtomizeWorkflow.php', 'DivinerAtomizer' => 'applications/diviner/atomizer/DivinerAtomizer.php', 'DivinerDefaultRenderer' => 'applications/diviner/renderer/DivinerDefaultRenderer.php', + 'DivinerDiskCache' => 'applications/diviner/cache/DivinerDiskCache.php', 'DivinerFileAtomizer' => 'applications/diviner/atomizer/DivinerFileAtomizer.php', 'DivinerGenerateWorkflow' => 'applications/diviner/workflow/DivinerGenerateWorkflow.php', 'DivinerListController' => 'applications/diviner/controller/DivinerListController.php', + 'DivinerPublishCache' => 'applications/diviner/cache/DivinerPublishCache.php', 'DivinerPublisher' => 'applications/diviner/publisher/DivinerPublisher.php', 'DivinerRenderer' => 'applications/diviner/renderer/DivinerRenderer.php', 'DivinerStaticPublisher' => 'applications/diviner/publisher/DivinerStaticPublisher.php', @@ -1965,11 +1967,13 @@ phutil_register_library_map(array( 'DiffusionURITestCase' => 'ArcanistPhutilTestCase', 'DiffusionView' => 'AphrontView', 'DivinerArticleAtomizer' => 'DivinerAtomizer', + 'DivinerAtomCache' => 'DivinerDiskCache', 'DivinerAtomizeWorkflow' => 'DivinerWorkflow', 'DivinerDefaultRenderer' => 'DivinerRenderer', 'DivinerFileAtomizer' => 'DivinerAtomizer', 'DivinerGenerateWorkflow' => 'DivinerWorkflow', 'DivinerListController' => 'PhabricatorController', + 'DivinerPublishCache' => 'DivinerDiskCache', 'DivinerStaticPublisher' => 'DivinerPublisher', 'DivinerWorkflow' => 'PhutilArgumentWorkflow', 'DrydockAllocatorWorker' => 'PhabricatorWorker', diff --git a/src/applications/diviner/cache/DivinerAtomCache.php b/src/applications/diviner/cache/DivinerAtomCache.php index 382c977e56..4b3ca32b79 100644 --- a/src/applications/diviner/cache/DivinerAtomCache.php +++ b/src/applications/diviner/cache/DivinerAtomCache.php @@ -1,8 +1,6 @@ setCacheDirectory($cache_directory); - $profiled_cache = id(new PhutilKeyValueCacheProfiler($dir_cache)) - ->setProfiler(PhutilServiceProfiler::getInstance()) - ->setName('diviner-atom-cache'); - $this->cache = $profiled_cache; - } - - private function getCache() { - return $this->cache; + return parent::__construct($cache_directory, 'diviner-atom-cache'); } public function delete() { - $this->getCache()->destroyCache(); + parent::delete(); $this->fileHashMap = null; $this->atomMap = null; $this->atoms = array(); @@ -36,24 +25,6 @@ final class DivinerAtomCache { return $this; } - /** - * Convert a long-form hash key like `ccbbaaaaaaaaaaaaaaaaaaaaaaaaaaaaN` into - * a shortened directory form, like `cc/bb/aaaaaaaaN`. In conjunction with - * @{class:PhutilKeyValueCacheDirectory}, this gives us nice directories - * inside .divinercache instead of a million hash files with huge names at - * top level. - */ - private function getHashKey($hash) { - return implode( - '/', - array( - substr($hash, 0, 2), - substr($hash, 2, 2), - substr($hash, 4, 8), - )); - } - - /* -( File Hash Map )------------------------------------------------------ */ diff --git a/src/applications/diviner/cache/DivinerDiskCache.php b/src/applications/diviner/cache/DivinerDiskCache.php new file mode 100644 index 0000000000..1bced54e83 --- /dev/null +++ b/src/applications/diviner/cache/DivinerDiskCache.php @@ -0,0 +1,42 @@ +setCacheDirectory($cache_directory); + $profiled_cache = id(new PhutilKeyValueCacheProfiler($dir_cache)) + ->setProfiler(PhutilServiceProfiler::getInstance()) + ->setName($name); + $this->cache = $profiled_cache; + } + + protected function getCache() { + return $this->cache; + } + + public function delete() { + $this->getCache()->destroyCache(); + return $this; + } + + /** + * Convert a long-form hash key like `ccbbaaaaaaaaaaaaaaaaaaaaaaaaaaaaN` into + * a shortened directory form, like `cc/bb/aaaaaaaaN`. In conjunction with + * @{class:PhutilKeyValueCacheDirectory}, this gives us nice directories + * inside .divinercache instead of a million hash files with huge names at + * top level. + */ + protected function getHashKey($hash) { + return implode( + '/', + array( + substr($hash, 0, 2), + substr($hash, 2, 2), + substr($hash, 4, 8), + )); + } + +} diff --git a/src/applications/diviner/cache/DivinerPublishCache.php b/src/applications/diviner/cache/DivinerPublishCache.php new file mode 100644 index 0000000000..381a032fc4 --- /dev/null +++ b/src/applications/diviner/cache/DivinerPublishCache.php @@ -0,0 +1,45 @@ +pathMap === null) { + $this->pathMap = $this->getCache()->getKey('path', array()); + } + return $this->pathMap; + } + + public function writePathMap() { + $this->getCache()->setKey('path', $this->getPathMap()); + } + + public function getAtomPathsFromCache($hash) { + return idx($this->getPathMap(), $hash, array()); + } + + public function removeAtomPathsFromCache($hash) { + $map = $this->getPathMap(); + unset($map[$hash]); + $this->pathMap = $map; + return $this; + } + + public function addAtomPathsToCache($hash, array $paths) { + $map = $this->getPathMap(); + $map[$hash] = $paths; + $this->pathMap = $map; + return $this; + } + + +} diff --git a/src/applications/diviner/publisher/DivinerPublisher.php b/src/applications/diviner/publisher/DivinerPublisher.php index f586cb88ba..98837a2422 100644 --- a/src/applications/diviner/publisher/DivinerPublisher.php +++ b/src/applications/diviner/publisher/DivinerPublisher.php @@ -113,8 +113,11 @@ abstract class DivinerPublisher { $deleted = array_diff_key($existing_map, $hashes_map); $created = array_diff_key($hashes_map, $existing_map); - $this->createDocumentsByHash(array_keys($created)); + echo pht('Deleting %d documents.', count($deleted))."\n"; $this->deleteDocumentsByHash(array_keys($deleted)); + + echo pht('Creating %d documents.', count($created))."\n"; + $this->createDocumentsByHash(array_keys($created)); } protected function shouldGenerateDocumentForAtom(DivinerAtom $atom) { diff --git a/src/applications/diviner/publisher/DivinerStaticPublisher.php b/src/applications/diviner/publisher/DivinerStaticPublisher.php index baae51f26c..1f7bb08edd 100644 --- a/src/applications/diviner/publisher/DivinerStaticPublisher.php +++ b/src/applications/diviner/publisher/DivinerStaticPublisher.php @@ -2,25 +2,86 @@ final class DivinerStaticPublisher extends DivinerPublisher { + private $publishCache; + + private function getPublishCache() { + if (!$this->publishCache) { + $dir = implode( + DIRECTORY_SEPARATOR, + array( + $this->getConfig('root'), + '.divinercache', + $this->getConfig('name'), + 'static', + )); + $this->publishCache = new DivinerPublishCache($dir); + } + + return $this->publishCache; + } + protected function loadAllPublishedHashes() { - return array(); + return array_keys($this->getPublishCache()->getPathMap()); } protected function deleteDocumentsByHash(array $hashes) { - return; + $root = $this->getConfig('root'); + $cache = $this->getPublishCache(); + + foreach ($hashes as $hash) { + $paths = $cache->getAtomPathsFromCache($hash); + foreach ($paths as $path) { + $abs = $root.DIRECTORY_SEPARATOR.$path; + Filesystem::remove($abs); + + // If the parent directory is now empty, clean it up. + $dir = dirname($abs); + while (true) { + if (!Filesystem::isDescendant($dir, $root)) { + // Directory is outside of the root. + break; + } + if (Filesystem::listDirectory($dir)) { + // Directory is not empty. + break; + } + + Filesystem::remove($dir); + $dir = dirname($dir); + } + } + + $cache->removeAtomPathsFromCache($hash); + $cache->deleteRenderCache($hash); + } + + $cache->writePathMap(); } protected function createDocumentsByHash(array $hashes) { + $indexes = array(); + foreach ($hashes as $hash) { $atom = $this->getAtomFromGraphHash($hash); - if (!$this->shouldGenerateDocumentForAtom($atom)) { - continue; + $paths = array(); + if ($this->shouldGenerateDocumentForAtom($atom)) { + $content = $this->getRenderer()->renderAtom($atom); + + $this->writeDocument($atom, $content); + + $paths[] = $this->getAtomRelativePath($atom); + if ($this->getAtomSimilarIndex($atom) !== null) { + $index = dirname($this->getAtomRelativePath($atom)).'index.html'; + $indexes[$index] = $atom; + $paths[] = $index; + } } - $content = $this->getRenderer()->renderAtom($atom); - $this->writeDocument($atom, $content); + $this->getPublishCache()->addAtomPathsToCache($hash, $paths); } + + $this->getPublishCache()->writePathMap(); } private function writeDocument(DivinerAtom $atom, $content) { @@ -32,6 +93,8 @@ final class DivinerStaticPublisher extends DivinerPublisher { } Filesystem::writeFile($path.'index.html', $content); + + return $this; } private function getAtomRelativePath(DivinerAtom $atom) {