Remove XHPAST from arcanist.

Summary: see corresponding commit in libphutil

Test Plan: linted and analyzed both libraries, got more sensible build behavior

Reviewers:

CC:

.gitignore

@@ -1,11 +1,2 @@
 .DS_Store
 ._*
-*.a
-*.o
-/support/xhpast/xhpast
-/src/staticanalysis/parsers/xhpast/bin/xhpast
-parser.yacc.cpp
-parser.yacc.hpp
-scanner.lex.cpp
-scanner.lex.hpp
-parser.yacc.output

scripts/phutil_analyzer.php

@@ -48,9 +48,13 @@ if ($argc != 2) {
 phutil_require_module('phutil', 'filesystem');
 $dir = Filesystem::resolvePath($argv[1]);
 
-phutil_require_module('arcanist', 'staticanalysis/parsers/xhpast/bin');
+phutil_require_module('phutil', 'parser/xhpast/bin');
+phutil_require_module('phutil', 'parser/xhpast/api/tree');
+
 phutil_require_module('arcanist', 'lint/linter/phutilmodule');
 phutil_require_module('arcanist', 'lint/message');
+phutil_require_module('arcanist', 'staticanalysis/parsers/phutilmodule');
+
 
 $data = array();
 $futures = array();
@@ -62,8 +66,6 @@ foreach (Filesystem::listDirectory($dir, $hidden_files = false) as $file) {
   $futures[$file] = xhpast_get_parser_future($data[$file]);
 }
 
-phutil_require_module('arcanist', 'staticanalysis/parsers/xhpast/api/tree');
-phutil_require_module('arcanist', 'staticanalysis/parsers/phutilmodule');
 
 $requirements = new PhutilModuleRequirements();
 $requirements->addBuiltins($builtin);

src/__phutil_library_map__.php

@@ -64,17 +64,9 @@ phutil_register_library_map(array(
     'PhutilModuleRequirements' => 'staticanalysis/parsers/phutilmodule',
     'PhutilUnitTestEngine' => 'unit/engine/phutil',
     'UnitTestableArcanistLintEngine' => 'lint/engine/test',
-    'XHPASTNode' => 'staticanalysis/parsers/xhpast/api/node',
-    'XHPASTNodeList' => 'staticanalysis/parsers/xhpast/api/list',
-    'XHPASTSyntaxErrorException' => 'staticanalysis/parsers/xhpast/api/exception',
-    'XHPASTToken' => 'staticanalysis/parsers/xhpast/api/token',
-    'XHPASTTree' => 'staticanalysis/parsers/xhpast/api/tree',
   ),
   'function' =>
   array(
-    'xhp_parser_node_constants' => 'staticanalysis/parsers/xhpast/constants',
-    'xhpast_get_parser_future' => 'staticanalysis/parsers/xhpast/bin',
-    'xhpast_parser_token_constants' => 'staticanalysis/parsers/xhpast/constants',
   ),
   'requires_class' =>
   array(

src/lint/linter/phutilmodule/ArcanistPhutilModuleLinter.php

@@ -90,6 +90,12 @@ class ArcanistPhutilModuleLinter extends ArcanistLinter {
 
   public function willLintPaths(array $paths) {
 
+    if ($paths) {
+      if (!xhpast_is_available()) {
+        throw new Exception(xhpast_get_build_instructions());
+      }
+    }
+
     $modules = array();
     $moduleinfo = array();
 

src/lint/linter/phutilmodule/__init__.php

@@ -14,6 +14,7 @@ phutil_require_module('phutil', 'filesystem');
 phutil_require_module('phutil', 'future');
 phutil_require_module('phutil', 'future/exec');
 phutil_require_module('phutil', 'moduleutils');
+phutil_require_module('phutil', 'parser/xhpast/bin');
 
 
 phutil_require_source('ArcanistPhutilModuleLinter.php');

src/lint/linter/xhpast/__init__.php

@@ -8,8 +8,8 @@
 
 phutil_require_module('arcanist', 'lint/linter/base');
 phutil_require_module('arcanist', 'lint/severity');
-phutil_require_module('arcanist', 'staticanalysis/parsers/xhpast/api/tree');
+phutil_require_module('phutil', 'parser/xhpast/api/tree');
-phutil_require_module('arcanist', 'staticanalysis/parsers/xhpast/bin');
+phutil_require_module('phutil', 'parser/xhpast/bin');
 
 phutil_require_module('phutil', 'utils');
 

src/staticanalysis/parsers/xhpast/api/exception/XHPASTSyntaxErrorException.php

@@ -1,32 +0,0 @@
-<?php
-
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-class XHPASTSyntaxErrorException extends Exception {
-
-  protected $errorLine;
-
-  public function __construct($line, $message) {
-    $this->errorLine = $line;
-    parent::__construct($message);
-  }
-
-  public function getErrorLine() {
-    return $this->errorLine;
-  }
-
-}

src/staticanalysis/parsers/xhpast/api/exception/__init__.php

@@ -1,10 +0,0 @@
-<?php
-/**
- * This file is automatically generated. Lint this module to rebuild it.
- * @generated
- */
-
-
-
-
-phutil_require_source('XHPASTSyntaxErrorException.php');

src/staticanalysis/parsers/xhpast/api/list/XHPASTNodeList.php

@@ -1,152 +0,0 @@
-<?php
-
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-class XHPASTNodeList implements Iterator, Countable {
-
-  protected $list;
-  protected $tree;
-  protected $ids;
-  protected $pos;
-
-  public function count() {
-    return count($this->ids);
-  }
-
-  public function current() {
-    return $this->list[$this->key()];
-  }
-
-  public function rewind() {
-    $this->pos = 0;
-  }
-
-  public function valid() {
-    return $this->pos < count($this->ids);
-  }
-
-  public function next() {
-    $this->pos++;
-  }
-
-  public function key() {
-    return $this->ids[$this->pos];
-  }
-
-  public static function newFromTreeAndNodes(XHPASTTree $tree, array $nodes) {
-    $obj = new XHPASTNodeList();
-    $obj->tree = $tree;
-    $obj->list = $nodes;
-    $obj->ids  = array_keys($nodes);
-    return $obj;
-  }
-
-  public static function newFromTree(XHPASTTree $tree) {
-    $obj = new XHPASTNodeList();
-    $obj->tree = $tree;
-    $obj->list = array(0 => $tree->getRootNode());
-    $obj->ids = array(0 => 0);
-    return $obj;
-  }
-
-  protected function __construct() {
-
-  }
-
-  public function getDescription() {
-    if (empty($this->list)) {
-      return 'an empty node list';
-    }
-
-    $desc = array();
-    $desc[] = "a list of ".count($this->list)." nodes:";
-    foreach ($this->list as $node) {
-      $desc[] = '  '.$node->getDescription().";";
-    }
-
-    return implode("\n", $desc);
-  }
-
-
-
-  protected function newList(array $nodes) {
-    return XHPASTNodeList::newFromTreeAndNodes(
-      $this->tree,
-      $nodes);
-  }
-
-  public function selectDescendantsOfType($type_name) {
-    $results = array();
-    foreach ($this->list as $id => $node) {
-      $results += $node->selectDescendantsOfType($type_name)->getRawNodes();
-    }
-    return $this->newList($results);
-  }
-
-  public function selectDescendantsOfTypes(array $type_names) {
-    $results = array();
-    foreach ($type_names as $type_name) {
-      foreach ($this->list as $id => $node) {
-        $results += $node->selectDescendantsOfType($type_name)->getRawNodes();
-      }
-    }
-    return $this->newList($results);
-  }
-
-  public function getChildrenByIndex($index) {
-    $results = array();
-    foreach ($this->list as $id => $node) {
-      $child = $node->getChildByIndex($index);
-      $results[$child->getID()] = $child;
-    }
-    return $this->newList($results);
-  }
-
-  public function add(XHPASTNodeList $list) {
-    foreach ($list->list as $id => $node) {
-      $this->list[$id] = $node;
-    }
-    $this->ids = array_keys($this->list);
-    return $this;
-  }
-
-
-  protected function executeSelectDescendantsOfType($node, $type) {
-    $results = array();
-    foreach ($node->getChildren() as $id => $child) {
-      if ($child->getTypeID() == $type) {
-        $results[$id] = $child;
-      } else {
-        $results += $this->executeSelectDescendantsOfType($child, $type);
-      }
-    }
-    return $results;
-  }
-
-  public function getTokens() {
-    $tokens = array();
-    foreach ($this->list as $node) {
-      $tokens += $node->getTokens();
-    }
-    return $tokens;
-  }
-
-  public function getRawNodes() {
-    return $this->list;
-  }
-
-}

src/staticanalysis/parsers/xhpast/api/list/__init__.php

@@ -1,19 +0,0 @@
-<?php
-
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-phutil_require_source('XHPASTNodeList.php');

src/staticanalysis/parsers/xhpast/api/node/XHPASTNode.php

@@ -1,231 +0,0 @@
-<?php
-
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-class XHPASTNode {
-
-  protected $id;
-  protected $l;
-  protected $r;
-  protected $typeID;
-  protected $children = array();
-  protected $tree;
-  protected $parentNode;
-
-  public function __construct($id, array $data, XHPASTTree $tree) {
-    $this->id = $id;
-    $this->typeID = $data[0];
-    $this->l = idx($data, 1, -1);
-    $this->r = idx($data, 2, -1);
-    $this->tree = $tree;
-  }
-
-  public function setParentNode($parent_node) {
-    $this->parentNode = $parent_node;
-    return $this;
-  }
-
-  public function getParentNode() {
-    return $this->parentNode;
-  }
-
-  public function setChildren(array $children) {
-    $this->children = $children;
-    return $this;
-  }
-
-  public function getID() {
-    return $this->id;
-  }
-
-  public function getTypeID() {
-    return $this->typeID;
-  }
-
-  public function getTypeName() {
-    static $map;
-    if (empty($map)) {
-      $map = xhp_parser_node_constants();
-    }
-
-    $type_id = $this->getTypeID();
-    if (empty($map[$type_id])) {
-      throw new Exception("No type name for node type ID '{$type_id}'.");
-    }
-
-    return $map[$type_id];
-  }
-
-  public function getChildren() {
-    return $this->children;
-  }
-
-  public function getChildOfType($index, $type) {
-    $child = $this->getChildByIndex($index);
-    if ($child->getTypeName() != $type) {
-      throw new Exception(
-        "Child in position '{$index}' is not of type '{$type}': ".
-        $this->getDescription());
-    }
-
-    return $child;
-  }
-
-  public function getChildByIndex($index) {
-    $child = idx(array_values($this->children), $index);
-    if (!$child) {
-      throw new Exception(
-        "No child with index '{$index}'.");
-    }
-    return $child;
-  }
-
-  public function selectDescendantsOfType($type_name) {
-    $type = $this->getTypeIDFromTypeName($type_name);
-    return XHPASTNodeList::newFromTreeAndNodes(
-      $this->tree,
-      $this->executeSelectDescendantsOfType($this, $type));
-  }
-
-  protected function executeSelectDescendantsOfType($node, $type) {
-    $results = array();
-    foreach ($node->getChildren() as $id => $child) {
-      if ($child->getTypeID() == $type) {
-        $results[$id] = $child;
-      }
-      $results += $this->executeSelectDescendantsOfType($child, $type);
-    }
-    return $results;
-  }
-
-  public function getTokens() {
-    if ($this->l == -1 || $this->r == -1) {
-      return array();
-    }
-    $tokens = $this->tree->getRawTokenStream();
-    $result = array();
-    foreach (range($this->l, $this->r) as $token_id) {
-      $result[$token_id] = $tokens[$token_id];
-    }
-    return $result;
-  }
-
-  public function getConcreteString() {
-    $values = array();
-    foreach ($this->getTokens() as $token) {
-      $values[] = $token->getValue();
-    }
-    return implode('', $values);
-  }
-
-  public function getStringLiteralValue() {
-    // TODO: This function should accommodate concatenation of literals and
-    // return 'null' if the literal contains variables.
-
-    if ($this->getTypeName() != 'n_STRING_SCALAR') {
-      return null;
-    }
-    $value = $this->getConcreteString();
-    $value = substr($value, 1, -1);
-    $value = stripcslashes($value);
-    return $value;
-  }
-
-  public function getSemanticString() {
-    $tokens = $this->getTokens();
-    foreach ($tokens as $id => $token) {
-      if ($token->isComment()) {
-        unset($tokens[$id]);
-      }
-    }
-    return implode('', mpull($tokens, 'getValue'));
-  }
-
-  public function getDescription() {
-    $concrete = $this->getConcreteString();
-    if (strlen($concrete) > 75) {
-      $concrete = substr($concrete, 0, 36).'...'.substr($concrete, -36);
-    }
-
-    $concrete = addcslashes($concrete, "\\\n\"");
-
-    return 'a node of type '.$this->getTypeName().': "'.$concrete.'"';
-  }
-
-  protected function getTypeIDFromTypeName($type_name) {
-    static $node_types;
-    if (empty($node_types)) {
-      $node_types = xhp_parser_node_constants();
-      $node_types = array_flip($node_types);
-    }
-
-    if (empty($node_types[$type_name])) {
-      throw new Exception("Unknown XHPAST Node type name '{$type_name}'!");
-    }
-
-    return $node_types[$type_name];
-  }
-
-  public function getOffset() {
-    $first_token = idx($this->tree->getRawTokenStream(), $this->l);
-    if (!$first_token) {
-      return null;
-    }
-    return $first_token->getOffset();
-  }
-
-  public function isStaticScalar() {
-    return ($this->getTypeName() == 'n_STRING_SCALAR' ||
-            $this->getTypeName() == 'n_NUMERIC_SCALAR');
-  }
-
-  public function getSurroundingNonsemanticTokens() {
-    $before = array();
-    $after  = array();
-
-    $tokens = $this->tree->getRawTokenStream();
-
-    if ($this->l != -1) {
-      $before = $tokens[$this->l]->getNonsemanticTokensBefore();
-    }
-
-    if ($this->r != -1) {
-      $after = $tokens[$this->r]->getNonsemanticTokensAfter();
-    }
-
-    return array($before, $after);
-  }
-
-  public function getDocblockToken() {
-    if ($this->l == -1) {
-      return null;
-    }
-    $tokens = $this->tree->getRawTokenStream();
-
-    for ($ii = $this->l - 1; $ii >= 0; $ii--) {
-      if ($tokens[$ii]->getTypeName() == 'T_DOC_COMMENT') {
-        return $tokens[$ii];
-      }
-      if (!$tokens[$ii]->isAnyWhitespace()) {
-        return null;
-      }
-    }
-
-    return null;
-  }
-
-}

src/staticanalysis/parsers/xhpast/api/node/__init__.php

@@ -1,23 +0,0 @@
-<?php
-
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-phutil_require_module('phutil', 'utils');
-phutil_require_module('arcanist', 'staticanalysis/parsers/xhpast/constants');
-phutil_require_module('arcanist', 'staticanalysis/parsers/xhpast/api/list');
-
-phutil_require_source('XHPASTNode.php');

src/staticanalysis/parsers/xhpast/api/token/XHPASTToken.php

@@ -1,102 +0,0 @@
-<?php
-
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-class XHPASTToken {
-
-  protected $id;
-  protected $typeID;
-  protected $value;
-  protected $offset;
-
-  public function __construct($id, $type, $value, $offset, XHPASTTree $tree) {
-    $this->id = $id;
-    $this->typeID = $type;
-    $this->offset = $offset;
-    $this->value = $value;
-    $this->tree = $tree;
-  }
-
-  public function getTypeID() {
-    return $this->typeID;
-  }
-
-  public function getTypeName() {
-    static $map;
-    if (empty($map)) {
-      $map = xhpast_parser_token_constants();
-    }
-
-    $type_id = $this->getTypeID();
-
-    if ($type_id <= 255) {
-      return chr($type_id);
-    }
-
-    if (empty($map[$type_id])) {
-      throw new Exception("No type name for token type ID '{$type_id}'.");
-    }
-
-    return $map[$type_id];
-  }
-
-  public function getValue() {
-    return $this->value;
-  }
-
-  public function getOffset() {
-    return $this->offset;
-  }
-
-  public function isComment() {
-    return ($this->getTypeName() == 'T_COMMENT' ||
-            $this->getTypeName() == 'T_DOC_COMMENT');
-  }
-
-  public function isAnyWhitespace() {
-    return ($this->getTypeName() == 'T_WHITESPACE' ||
-            $this->getTypeName() == 'T_XHP_WHITESPACE');
-  }
-
-  public function isSemantic() {
-    return !($this->isComment() || $this->isAnyWhitespace());
-  }
-
-  public function getNonsemanticTokensBefore() {
-    $tokens = $this->tree->getRawTokenStream();
-    $result = array();
-    $ii = $this->id - 1;
-    while ($ii >= 0 && !$tokens[$ii]->isSemantic()) {
-      $result[$ii] = $tokens[$ii];
-      --$ii;
-    }
-    return array_reverse($result);
-  }
-
-  public function getNonsemanticTokensAfter() {
-    $tokens = $this->tree->getRawTokenStream();
-    $result = array();
-    $ii = $this->id + 1;
-    while ($ii < count($tokens) && !$tokens[$ii]->isSemantic()) {
-      $result[$ii] = $tokens[$ii];
-      ++$ii;
-    }
-    return $result;
-  }
-
-
-}

src/staticanalysis/parsers/xhpast/api/token/__init__.php

@@ -1,12 +0,0 @@
-<?php
-/**
- * This file is automatically generated. Lint this module to rebuild it.
- * @generated
- */
-
-
-
-phutil_require_module('arcanist', 'staticanalysis/parsers/xhpast/constants');
-
-
-phutil_require_source('XHPASTToken.php');

src/staticanalysis/parsers/xhpast/api/tree/XHPASTTree.php

@@ -1,117 +0,0 @@
-<?php
-
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-class XHPASTTree {
-
-  protected $tree = array();
-  protected $stream = array();
-
-  public static function newFromData($php_source) {
-    $future = xhpast_get_parser_future($php_source);
-    return self::newFromDataAndResolvedExecFuture($future->resolve());
-  }
-
-  public static function newFromDataAndResolvedExecFuture(
-    $php_source,
-    array $resolved) {
-
-    list($err, $stdout, $stderr) = $resolved;
-    if ($err) {
-      if ($err == 1) {
-        $matches = null;
-        $is_syntax = preg_match(
-          '/^XHPAST Parse Error: (.*) on line (\d+)/',
-          $stderr,
-          $matches);
-        if ($is_syntax) {
-          throw new XHPASTSyntaxErrorException($matches[2], $stderr);
-        }
-      }
-      throw new Exception("XHPAST failed to parse file data {$err}: {$stderr}");
-    }
-
-    $data = json_decode($stdout, true);
-    if (!is_array($data)) {
-      throw new Exception("XHPAST: failed to decode tree.");
-    }
-
-    return new XHPASTTree($data['tree'], $data['stream'], $php_source);
-  }
-
-  public function __construct(array $tree, array $stream, $source) {
-    $ii = 0;
-    $offset = 0;
-    foreach ($stream as $token) {
-      $this->stream[$ii] = new XHPASTToken(
-        $ii,
-        $token[0],
-        substr($source, $offset, $token[1]),
-        $offset,
-        $this);
-      $offset += $token[1];
-      ++$ii;
-    }
-
-    $this->buildTree(array($tree));
-  }
-
-  public function getRootNode() {
-    return $this->tree[0];
-  }
-
-  protected function buildTree(array $tree) {
-    $ii = count($this->tree);
-    $nodes = array();
-    foreach ($tree as $node) {
-      $this->tree[$ii] = new XHPASTNode($ii, $node, $this);
-      $nodes[$ii] = $node;
-      ++$ii;
-    }
-    foreach ($nodes as $node_id => $node) {
-      if (isset($node[3])) {
-        $children = $this->buildTree($node[3]);
-        foreach ($children as $child) {
-          $child->setParentNode($this->tree[$node_id]);
-        }
-        $this->tree[$node_id]->setChildren($children);
-      }
-    }
-    return array_select_keys($this->tree, array_keys($nodes));
-  }
-
-  public function getRawTokenStream() {
-    return $this->stream;
-  }
-
-  public function renderAsText() {
-    return $this->executeRenderAsText(array($this->getRootNode()), 0);
-  }
-
-  protected function executeRenderAsText($list, $depth) {
-    $return = '';
-    foreach ($list as $node) {
-      if ($depth) {
-        $return .= str_repeat('  ', $depth);
-      }
-      $return .= $node->getDescription()."\n";
-      $return .= $this->executeRenderAsText($node->getChildren(), $depth + 1);
-    }
-    return $return;
-  }
-
-}

src/staticanalysis/parsers/xhpast/api/tree/__init__.php

@@ -1,17 +0,0 @@
-<?php
-/**
- * This file is automatically generated. Lint this module to rebuild it.
- * @generated
- */
-
-
-
-phutil_require_module('arcanist', 'staticanalysis/parsers/xhpast/api/exception');
-phutil_require_module('arcanist', 'staticanalysis/parsers/xhpast/api/node');
-phutil_require_module('arcanist', 'staticanalysis/parsers/xhpast/api/token');
-phutil_require_module('arcanist', 'staticanalysis/parsers/xhpast/bin');
-
-phutil_require_module('phutil', 'utils');
-
-
-phutil_require_source('XHPASTTree.php');

src/staticanalysis/parsers/xhpast/bin/__init__.php

@@ -1,21 +0,0 @@
-<?php
-
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-phutil_require_module('phutil', 'future/exec');
-
-phutil_require_source('xhpast_parse.php');

src/staticanalysis/parsers/xhpast/bin/xhpast_parse.php

@@ -1,36 +0,0 @@
-<?php
-
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-function xhpast_get_parser_future($data) {
-  static $bin_path;
-  if (empty($bin_path)) {
-    $root = dirname(__FILE__);
-    $bin_path = $root.'/xhpast';
-  }
-
-  if (!file_exists($bin_path)) {
-    execx(
-      '(cd %s && make && make install)',
-      dirname(__FILE__).'/../../../../../support/xhpast');
-  }
-
-  $future = new ExecFuture($bin_path);
-  $future->write($data);
-
-  return $future;
-}

src/staticanalysis/parsers/xhpast/constants/__init__.php

@@ -1,20 +0,0 @@
-<?php
-
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-phutil_require_source('parser_nodes.php');
-phutil_require_source('parser_tokens.php');

src/staticanalysis/parsers/xhpast/constants/parser_nodes.php

@@ -1,139 +0,0 @@
-<?php
-
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-function xhp_parser_node_constants() {
-  return array(
-    9000 => 'n_PROGRAM',
-    9001 => 'n_SYMBOL_NAME',
-    9002 => 'n_HALT_COMPILER',
-    9003 => 'n_NAMESPACE',
-    9004 => 'n_STATEMENT',
-    9005 => 'n_EMPTY',
-    9006 => 'n_STATEMENT_LIST',
-    9007 => 'n_OPEN_TAG',
-    9008 => 'n_CLOSE_TAG',
-    9009 => 'n_USE_LIST',
-    9010 => 'n_USE',
-    9011 => 'n_CONSTANT_DECLARATION_LIST',
-    9012 => 'n_CONSTANT_DECLARATION',
-    9013 => 'n_STRING',
-    9014 => 'n_LABEL',
-    9015 => 'n_CONDITION_LIST',
-    9016 => 'n_CONTROL_CONDITION',
-    9017 => 'n_IF',
-    9018 => 'n_ELSEIF',
-    9019 => 'n_ELSE',
-    9020 => 'n_WHILE',
-    9021 => 'n_DO_WHILE',
-    9022 => 'n_FOR',
-    9023 => 'n_FOR_EXPRESSION',
-    9024 => 'n_SWITCH',
-    9025 => 'n_BREAK',
-    9026 => 'n_CONTINUE',
-    9027 => 'n_RETURN',
-    9028 => 'n_GLOBAL_DECLARATION_LIST',
-    9029 => 'n_GLOBAL_DECLARATION',
-    9030 => 'n_STATIC_DECLARATION_LIST',
-    9031 => 'n_STATIC_DECLARATION',
-    9032 => 'n_ECHO_LIST',
-    9033 => 'n_ECHO',
-    9034 => 'n_INLINE_HTML',
-    9035 => 'n_UNSET_LIST',
-    9036 => 'n_UNSET',
-    9037 => 'n_FOREACH',
-    9038 => 'n_FOREACH_EXPRESSION',
-    9039 => 'n_THROW',
-    9040 => 'n_GOTO',
-    9041 => 'n_TRY',
-    9042 => 'n_CATCH_LIST',
-    9043 => 'n_CATCH',
-    9044 => 'n_DECLARE',
-    9045 => 'n_DECLARE_DECLARATION_LIST',
-    9046 => 'n_DECLARE_DECLARATION',
-    9047 => 'n_VARIABLE',
-    9048 => 'n_REFERENCE',
-    9049 => 'n_VARIABLE_REFERENCE',
-    9050 => 'n_FUNCTION_DECLARATION',
-    9051 => 'n_CLASS_DECLARATION',
-    9052 => 'n_CLASS_ATTRIBUTES',
-    9053 => 'n_EXTENDS',
-    9054 => 'n_EXTENDS_LIST',
-    9055 => 'n_IMPLEMENTS_LIST',
-    9056 => 'n_INTERFACE_DECLARATION',
-    9057 => 'n_CASE',
-    9058 => 'n_DEFAULT',
-    9059 => 'n_DECLARATION_PARAMETER_LIST',
-    9060 => 'n_DECLARATION_PARAMETER',
-    9061 => 'n_TYPE_NAME',
-    9062 => 'n_VARIABLE_VARIABLE',
-    9063 => 'n_CLASS_MEMBER_DECLARATION_LIST',
-    9064 => 'n_CLASS_MEMBER_DECLARATION',
-    9065 => 'n_CLASS_CONSTANT_DECLARATION_LIST',
-    9066 => 'n_CLASS_CONSTANT_DECLARATION',
-    9067 => 'n_METHOD_DECLARATION',
-    9068 => 'n_METHOD_MODIFIER_LIST',
-    9069 => 'n_FUNCTION_MODIFIER_LIST',
-    9070 => 'n_CLASS_MEMBER_MODIFIER_LIST',
-    9071 => 'n_EXPRESSION_LIST',
-    9072 => 'n_LIST',
-    9073 => 'n_ASSIGNMENT',
-    9074 => 'n_NEW',
-    9075 => 'n_UNARY_PREFIX_EXPRESSION',
-    9076 => 'n_UNARY_POSTFIX_EXPRESSION',
-    9077 => 'n_BINARY_EXPRESSION',
-    9078 => 'n_TERNARY_EXPRESSION',
-    9079 => 'n_CAST_EXPRESSION',
-    9080 => 'n_CAST',
-    9081 => 'n_OPERATOR',
-    9082 => 'n_ARRAY_LITERAL',
-    9083 => 'n_EXIT_EXPRESSION',
-    9084 => 'n_BACKTICKS_EXPRESSION',
-    9085 => 'n_LEXICAL_VARIABLE_LIST',
-    9086 => 'n_NUMERIC_SCALAR',
-    9087 => 'n_STRING_SCALAR',
-    9088 => 'n_MAGIC_SCALAR',
-    9089 => 'n_CLASS_STATIC_ACCESS',
-    9090 => 'n_CLASS_NAME',
-    9091 => 'n_MAGIC_CLASS_KEYWORD',
-    9092 => 'n_OBJECT_PROPERTY_ACCESS',
-    9093 => 'n_ARRAY_VALUE_LIST',
-    9094 => 'n_ARRAY_VALUE',
-    9095 => 'n_CALL_PARAMETER_LIST',
-    9096 => 'n_VARIABLE_EXPRESSION',
-    9097 => 'n_INCLUDE_FILE',
-    9098 => 'n_HEREDOC',
-    9099 => 'n_FUNCTION_CALL',
-    9100 => 'n_INDEX_ACCESS',
-    9101 => 'n_ASSIGNMENT_LIST',
-    9102 => 'n_METHOD_CALL',
-    9103 => 'n_XHP_TAG',
-    9104 => 'n_XHP_TAG_OPEN',
-    9105 => 'n_XHP_TAG_CLOSE',
-    9106 => 'n_XHP_TEXT',
-    9107 => 'n_XHP_EXPRESSION',
-    9108 => 'n_XHP_ATTRIBUTE_LIST',
-    9109 => 'n_XHP_ATTRIBUTE',
-    9110 => 'n_XHP_LITERAL',
-    9111 => 'n_XHP_ATTRIBUTE_LITERAL',
-    9112 => 'n_XHP_ATTRIBUTE_EXPRESSION',
-    9113 => 'n_XHP_NODE_LIST',
-    9114 => 'n_XHP_ENTITY',
-    9115 => 'n_CONCATENATION_LIST',
-    9116 => 'n_PARENTHETICAL_EXPRESSION',
-  );
-}

src/staticanalysis/parsers/xhpast/constants/parser_tokens.php

@@ -1,172 +0,0 @@
-<?php
-
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-function xhpast_parser_token_constants() {
-  return array(
-    258 => 'T_REQUIRE_ONCE',
-    259 => 'T_REQUIRE',
-    260 => 'T_EVAL',
-    261 => 'T_INCLUDE_ONCE',
-    262 => 'T_INCLUDE',
-    263 => 'T_LOGICAL_OR',
-    264 => 'T_LOGICAL_XOR',
-    265 => 'T_LOGICAL_AND',
-    266 => 'T_PRINT',
-    267 => 'T_SR_EQUAL',
-    268 => 'T_SL_EQUAL',
-    269 => 'T_XOR_EQUAL',
-    270 => 'T_OR_EQUAL',
-    271 => 'T_AND_EQUAL',
-    272 => 'T_MOD_EQUAL',
-    273 => 'T_CONCAT_EQUAL',
-    274 => 'T_DIV_EQUAL',
-    275 => 'T_MUL_EQUAL',
-    276 => 'T_MINUS_EQUAL',
-    277 => 'T_PLUS_EQUAL',
-    278 => 'T_BOOLEAN_OR',
-    279 => 'T_BOOLEAN_AND',
-    280 => 'T_IS_NOT_IDENTICAL',
-    281 => 'T_IS_IDENTICAL',
-    282 => 'T_IS_NOT_EQUAL',
-    283 => 'T_IS_EQUAL',
-    284 => 'T_IS_GREATER_OR_EQUAL',
-    285 => 'T_IS_SMALLER_OR_EQUAL',
-    286 => 'T_SR',
-    287 => 'T_SL',
-    288 => 'T_INSTANCEOF',
-    289 => 'T_UNSET_CAST',
-    290 => 'T_BOOL_CAST',
-    291 => 'T_OBJECT_CAST',
-    292 => 'T_ARRAY_CAST',
-    293 => 'T_BINARY_CAST',
-    294 => 'T_UNICODE_CAST',
-    295 => 'T_STRING_CAST',
-    296 => 'T_DOUBLE_CAST',
-    297 => 'T_INT_CAST',
-    298 => 'T_DEC',
-    299 => 'T_INC',
-    300 => 'T_CLONE',
-    301 => 'T_NEW',
-    302 => 'T_EXIT',
-    303 => 'T_IF',
-    304 => 'T_ELSEIF',
-    305 => 'T_ELSE',
-    306 => 'T_ENDIF',
-    307 => 'T_LNUMBER',
-    308 => 'T_DNUMBER',
-    309 => 'T_STRING',
-    310 => 'T_STRING_VARNAME',
-    311 => 'T_VARIABLE',
-    312 => 'T_NUM_STRING',
-    313 => 'T_INLINE_HTML',
-    314 => 'T_CHARACTER',
-    315 => 'T_BAD_CHARACTER',
-    316 => 'T_ENCAPSED_AND_WHITESPACE',
-    317 => 'T_CONSTANT_ENCAPSED_STRING',
-    318 => 'T_BACKTICKS_EXPR',
-    319 => 'T_ECHO',
-    320 => 'T_DO',
-    321 => 'T_WHILE',
-    322 => 'T_ENDWHILE',
-    323 => 'T_FOR',
-    324 => 'T_ENDFOR',
-    325 => 'T_FOREACH',
-    326 => 'T_ENDFOREACH',
-    327 => 'T_DECLARE',
-    328 => 'T_ENDDECLARE',
-    329 => 'T_AS',
-    330 => 'T_SWITCH',
-    331 => 'T_ENDSWITCH',
-    332 => 'T_CASE',
-    333 => 'T_DEFAULT',
-    334 => 'T_BREAK',
-    335 => 'T_CONTINUE',
-    336 => 'T_GOTO',
-    337 => 'T_FUNCTION',
-    338 => 'T_CONST',
-    339 => 'T_RETURN',
-    340 => 'T_TRY',
-    341 => 'T_CATCH',
-    342 => 'T_THROW',
-    343 => 'T_USE',
-    344 => 'T_GLOBAL',
-    345 => 'T_PUBLIC',
-    346 => 'T_PROTECTED',
-    347 => 'T_PRIVATE',
-    348 => 'T_FINAL',
-    349 => 'T_ABSTRACT',
-    350 => 'T_STATIC',
-    351 => 'T_VAR',
-    352 => 'T_UNSET',
-    353 => 'T_ISSET',
-    354 => 'T_EMPTY',
-    355 => 'T_HALT_COMPILER',
-    356 => 'T_CLASS',
-    357 => 'T_INTERFACE',
-    358 => 'T_EXTENDS',
-    359 => 'T_IMPLEMENTS',
-    360 => 'T_OBJECT_OPERATOR',
-    361 => 'T_DOUBLE_ARROW',
-    362 => 'T_LIST',
-    363 => 'T_ARRAY',
-    364 => 'T_CLASS_C',
-    365 => 'T_METHOD_C',
-    366 => 'T_FUNC_C',
-    367 => 'T_LINE',
-    368 => 'T_FILE',
-    369 => 'T_COMMENT',
-    370 => 'T_DOC_COMMENT',
-    371 => 'T_OPEN_TAG',
-    372 => 'T_OPEN_TAG_WITH_ECHO',
-    373 => 'T_OPEN_TAG_FAKE',
-    374 => 'T_CLOSE_TAG',
-    375 => 'T_WHITESPACE',
-    376 => 'T_START_HEREDOC',
-    377 => 'T_END_HEREDOC',
-    378 => 'T_HEREDOC',
-    379 => 'T_DOLLAR_OPEN_CURLY_BRACES',
-    380 => 'T_CURLY_OPEN',
-    381 => 'T_PAAMAYIM_NEKUDOTAYIM',
-    382 => 'T_BINARY_DOUBLE',
-    383 => 'T_BINARY_HEREDOC',
-    384 => 'T_NAMESPACE',
-    385 => 'T_NS_C',
-    386 => 'T_DIR',
-    387 => 'T_NS_SEPARATOR',
-    388 => 'T_XHP_WHITESPACE',
-    389 => 'T_XHP_TEXT',
-    390 => 'T_XHP_LT_DIV',
-    391 => 'T_XHP_LT_DIV_GT',
-    392 => 'T_XHP_ATTRIBUTE',
-    393 => 'T_XHP_CATEGORY',
-    394 => 'T_XHP_CHILDREN',
-    395 => 'T_XHP_ANY',
-    396 => 'T_XHP_EMPTY',
-    397 => 'T_XHP_PCDATA',
-    398 => 'T_XHP_COLON',
-    399 => 'T_XHP_HYPHEN',
-    400 => 'T_XHP_BOOLEAN',
-    401 => 'T_XHP_NUMBER',
-    402 => 'T_XHP_ARRAY',
-    403 => 'T_XHP_STRING',
-    404 => 'T_XHP_ENUM',
-    405 => 'T_XHP_FLOAT',
-    406 => 'T_XHP_REQUIRED',
-    407 => 'T_XHP_ENTITY',
-  );
-}

support/xhpast/Makefile

@@ -1,46 +0,0 @@
-ifdef DEBUG
-CPPFLAGS = -fPIC -ggdb -Wall -DDEBUG
-else
-CPPFLAGS = -fPIC -g -Wall -O3 -minline-all-stringops
-endif
-
-ifdef PROFILE
-CPPFLAGS += -pg
-endif
-
-all: xhpast
-
-clean:
-	-rm xhpast parser.yacc.cpp scanner.lex.cpp scanner.lex.hpp parser.yacc.output parser.yacc.hpp libxhpast.a *.o 2>/dev/null
-
-install: xhpast
-	cp xhpast ../../src/staticanalysis/parsers/xhpast/bin/
-
-parser.yacc.cpp: parser.y
-	bison --debug --verbose -d -o $@ $<
-
-parser.yacc.hpp: parser.yacc.cpp
-
-scanner.lex.cpp: scanner.l
-	`which flex35 2>/dev/null || which flex 2>/dev/null` \
-	  -C --header-file=scanner.lex.hpp -o $@ -d $<
-
-scanner.lex.hpp: scanner.lex.cpp
-
-node_names.hpp: generate_nodes.php
-	php -f generate_nodes.php
-
-%.o: %.cpp
-	$(CXX) -c $(CPPFLAGS) -o $@ $<
-
-parser.yacc.o: scanner.lex.hpp
-
-scanner.lex.o: parser.yacc.hpp node_names.hpp
-
-libxhpast.a: astnode.o scanner.lex.o parser.yacc.o
-	$(AR) -crs $@ $^
-
-xhpast: xhpast.cpp libxhpast.a
-	$(CXX) $(CPPFLAGS) -o $@ $^
-
-.PHONY: all clean tags

support/xhpast/ast.hpp

@@ -1,101 +0,0 @@
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#include <stdint.h>
-#include <deque>
-#include <stack>
-#include <string>
-
-#include "astnode.hpp"
-
-class yy_extra_type {
-  public:
-    yy_extra_type() {
-      lineno = 1;
-      terminated = false;
-      used = false;
-      short_tags = true;
-      asp_tags = false;
-      idx_expr = false;
-      include_debug = false;
-      expecting_xhp_class_statements = false;
-      list_size = 0;
-      colon_hack = false;
-      pushStack();
-    }
-
-    bool short_tags; // `short_open_tag` in php.ini
-    bool asp_tags; // `asp_tags` in php.ini
-    bool idx_expr; // allow code like `foo()['bar']`
-    bool include_debug; // include line numbers and file names in XHP object creation
-    size_t first_lineno; // line number before scanning the current token
-    size_t lineno; // current line number being scanned.
-    std::string error; // description of error (if terminated true)
-    bool terminated; // becomes true when the parser terminates with an error
-    bool used; // were any XHP-specific extensions found in this code?
-    int last_token; // the last token to be returned by the scanner
-    int insert_token; // insert this token without reading from buffer
-    size_t heredoc_yyleng; // last length of yytext while scannling
-    const char* heredoc_data; // where our heredoc data starts
-    std::string heredoc_label; // heredoc sentinel label
-    std::stack<int> curly_stack; // tokens appearing before a {
-    bool expecting_xhp_class_statements; // when we're one level deep in a class
-    bool old_expecting_xhp_class_statements; // store old value while inside class method
-    bool used_attributes; // did this class use the `attribute` keyword
-    unsigned int list_size;
-    bool colon_hack;
-
-    xhpast::token_list_t token_list;
-
-    /* Utility functions for checking proper tag closing */
-    bool haveTag() {
-      return !tag_stack.front().empty();
-    }
-    const std::string &peekTag() {
-      return tag_stack.front().front();
-    }
-    void pushTag(const std::string &tag) {
-      tag_stack.front().push_front(tag);
-    }
-    void popTag() {
-      tag_stack.front().pop_front();
-    }
-    void pushStack() {
-      tag_stack.push_front(std::deque<std::string>());
-    }
-    void popStack() {
-      tag_stack.pop_front();
-    }
-
-  protected:
-    std::deque<std::deque<std::string> > tag_stack;
-};
-
-#define YYSTYPE xhpast::Node *
-#define YY_HEADER_EXPORT_START_CONDITIONS
-#define YY_EXTRA_TYPE yy_extra_type*
-
-#include "parser.yacc.hpp"
-#ifndef FLEX_SCANNER
-  #include "scanner.lex.hpp"
-#endif
-
-int xhpparse(void*, YYSTYPE *);
-void xhp_new_push_state(int s, struct yyguts_t* yyg);
-void xhp_new_pop_state(struct yyguts_t* yyg);
-void xhp_set_state(int s, struct yyguts_t* yyg);

support/xhpast/astnode.cpp

@@ -1,17 +0,0 @@
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "astnode.hpp"

support/xhpast/astnode.hpp

@@ -1,126 +0,0 @@
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-#include <cstdio>
-#include <cstdlib>
-#include <list>
-#include <string>
-
-namespace xhpast {
-
-  class Token;
-  typedef std::list<Token *> token_list_t;
-
-  class Token {
-    
-    public: 
-      unsigned int type;
-      std::string value;
-      unsigned int lineno;
-      unsigned int n;
-      
-      Token(unsigned int type, char *value, unsigned int n) :
-        type(type),
-        value(value),
-        n(n) {
-      }
-  };
-
-  class Node;
-  typedef std::list<Node *> node_list_t;
-  
-  class Node {
-    public: 
-      unsigned int type;
-
-      int l_tok;
-      int r_tok;
-    
-      node_list_t children;
-    
-      
-      Node() : type(0), l_tok(-1), r_tok(-1) {};
-
-      Node(unsigned int type) : type(type), l_tok(-1), r_tok(-1) {};
-
-      Node(unsigned int type, int end_tok) :
-        type(type) {
-          this->l_tok = end_tok;
-          this->r_tok = end_tok;
-      }
-
-      Node(unsigned int type, int l_tok, int r_tok) :
-        type(type),
-        l_tok(l_tok),
-        r_tok(r_tok) {
-      
-      }
-
-      Node *appendChild(Node *node) {
-        this->children.push_back(node);
-        return this->setEnd(node);
-      }
-      
-      Node *appendChildren(Node *node) {
-        for (node_list_t::iterator ii = node->children.begin(); ii != node->children.end(); ++ii) {
-          this->children.push_back(*ii);
-          this->setEnd(*ii);
-        }
-        return this;
-      }
-      
-      Node *firstChild() {
-        return *(this->children.begin());
-      }
-      
-      Node *setType(unsigned int t) {
-        this->type = t;
-        return this;
-      }
-      
-      Node *setEnd(Node *n) {
-        if (!n) {
-          fprintf(stderr, "Trying to setEnd() a null node to one of type %d\n", this->type);
-          exit(1);
-        }
-        
-        if (n->r_tok != -1 && (n->r_tok > this->r_tok || (this->r_tok == -1))) {
-          this->r_tok = n->r_tok;
-        }
-        if (this->l_tok == -1) {
-          this->l_tok = n->l_tok;
-        }
-        return this;
-      }
-
-      Node *setBegin(Node *n) {
-        if (!n) {
-          fprintf(stderr, "Trying to setBegin() a null node to one of type %d\n", this->type);
-          exit(1);
-        }
-
-        if (n->l_tok != -1 && (n->l_tok < this->l_tok || (this->l_tok == -1))) {
-          this->l_tok = n->l_tok;
-        }
-        if (this->r_tok == -1) {
-          this->r_tok = n->r_tok;
-        }
-        return this;
-      }
-    
-  };
-}

support/xhpast/generate_nodes.php

@@ -1,165 +0,0 @@
-#!/usr/local/bin/php
-<?php
-
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-$nodes = <<<EONODES
-n_PROGRAM
-n_SYMBOL_NAME
-n_HALT_COMPILER
-n_NAMESPACE
-n_STATEMENT
-n_EMPTY
-n_STATEMENT_LIST
-n_OPEN_TAG
-n_CLOSE_TAG
-n_USE_LIST
-n_USE
-n_CONSTANT_DECLARATION_LIST
-n_CONSTANT_DECLARATION
-n_STRING
-n_LABEL
-n_CONDITION_LIST
-n_CONTROL_CONDITION
-n_IF
-n_ELSEIF
-n_ELSE
-n_WHILE
-n_DO_WHILE
-n_FOR
-n_FOR_EXPRESSION
-n_SWITCH
-n_BREAK
-n_CONTINUE
-n_RETURN
-n_GLOBAL_DECLARATION_LIST
-n_GLOBAL_DECLARATION
-n_STATIC_DECLARATION_LIST
-n_STATIC_DECLARATION
-n_ECHO_LIST
-n_ECHO
-n_INLINE_HTML
-n_UNSET_LIST
-n_UNSET
-n_FOREACH
-n_FOREACH_EXPRESSION
-n_THROW
-n_GOTO
-n_TRY
-n_CATCH_LIST
-n_CATCH
-n_DECLARE
-n_DECLARE_DECLARATION_LIST
-n_DECLARE_DECLARATION
-n_VARIABLE
-n_REFERENCE
-n_VARIABLE_REFERENCE
-n_FUNCTION_DECLARATION
-n_CLASS_DECLARATION
-n_CLASS_ATTRIBUTES
-n_EXTENDS
-n_EXTENDS_LIST
-n_IMPLEMENTS_LIST
-n_INTERFACE_DECLARATION
-n_CASE
-n_DEFAULT
-n_DECLARATION_PARAMETER_LIST
-n_DECLARATION_PARAMETER
-n_TYPE_NAME
-n_VARIABLE_VARIABLE
-n_CLASS_MEMBER_DECLARATION_LIST
-n_CLASS_MEMBER_DECLARATION
-n_CLASS_CONSTANT_DECLARATION_LIST
-n_CLASS_CONSTANT_DECLARATION
-n_METHOD_DECLARATION
-n_METHOD_MODIFIER_LIST
-n_FUNCTION_MODIFIER_LIST
-n_CLASS_MEMBER_MODIFIER_LIST
-n_EXPRESSION_LIST
-n_LIST
-n_ASSIGNMENT
-n_NEW
-n_UNARY_PREFIX_EXPRESSION
-n_UNARY_POSTFIX_EXPRESSION
-n_BINARY_EXPRESSION
-n_TERNARY_EXPRESSION
-n_CAST_EXPRESSION
-n_CAST
-n_OPERATOR
-n_ARRAY_LITERAL
-n_EXIT_EXPRESSION
-n_BACKTICKS_EXPRESSION
-n_LEXICAL_VARIABLE_LIST
-n_NUMERIC_SCALAR
-n_STRING_SCALAR
-n_MAGIC_SCALAR
-n_CLASS_STATIC_ACCESS
-n_CLASS_NAME
-n_MAGIC_CLASS_KEYWORD
-n_OBJECT_PROPERTY_ACCESS
-n_ARRAY_VALUE_LIST
-n_ARRAY_VALUE
-n_CALL_PARAMETER_LIST
-n_VARIABLE_EXPRESSION
-n_INCLUDE_FILE
-n_HEREDOC
-n_FUNCTION_CALL
-n_INDEX_ACCESS
-n_ASSIGNMENT_LIST
-n_METHOD_CALL
-n_XHP_TAG
-n_XHP_TAG_OPEN
-n_XHP_TAG_CLOSE
-n_XHP_TEXT
-n_XHP_EXPRESSION
-n_XHP_ATTRIBUTE_LIST
-n_XHP_ATTRIBUTE
-n_XHP_LITERAL
-n_XHP_ATTRIBUTE_LITERAL
-n_XHP_ATTRIBUTE_EXPRESSION
-n_XHP_NODE_LIST
-n_XHP_ENTITY
-n_CONCATENATION_LIST
-n_PARENTHETICAL_EXPRESSION
-EONODES;
-
-$seq = 9000;
-$map = array();
-foreach (explode("\n", trim($nodes)) as $node) {
-  $map[$node] = $seq++;
-}
-
-$hpp = '';
-foreach ($map as $node => $value) {
-  $hpp .= "#define {$node} {$value}\n";
-}
-file_put_contents('node_names.hpp', $hpp);
-echo "Wrote C++ definition.\n";
-
-$php =
-  "<?php\n\nfunction xhp_parser_node_constants() {\n".
-  "  return array(\n";
-foreach ($map as $node => $value) {
-  $php .= "    {$value} => '{$node}',\n";
-}
-$php .= "  );\n";
-$php .= "}\n";
-file_put_contents('parser_nodes.php', $php);
-echo "Wrote PHP definition.\n";
-
-
-

support/xhpast/node_names.hpp

@@ -1,117 +0,0 @@
-#define n_PROGRAM 9000
-#define n_SYMBOL_NAME 9001
-#define n_HALT_COMPILER 9002
-#define n_NAMESPACE 9003
-#define n_STATEMENT 9004
-#define n_EMPTY 9005
-#define n_STATEMENT_LIST 9006
-#define n_OPEN_TAG 9007
-#define n_CLOSE_TAG 9008
-#define n_USE_LIST 9009
-#define n_USE 9010
-#define n_CONSTANT_DECLARATION_LIST 9011
-#define n_CONSTANT_DECLARATION 9012
-#define n_STRING 9013
-#define n_LABEL 9014
-#define n_CONDITION_LIST 9015
-#define n_CONTROL_CONDITION 9016
-#define n_IF 9017
-#define n_ELSEIF 9018
-#define n_ELSE 9019
-#define n_WHILE 9020
-#define n_DO_WHILE 9021
-#define n_FOR 9022
-#define n_FOR_EXPRESSION 9023
-#define n_SWITCH 9024
-#define n_BREAK 9025
-#define n_CONTINUE 9026
-#define n_RETURN 9027
-#define n_GLOBAL_DECLARATION_LIST 9028
-#define n_GLOBAL_DECLARATION 9029
-#define n_STATIC_DECLARATION_LIST 9030
-#define n_STATIC_DECLARATION 9031
-#define n_ECHO_LIST 9032
-#define n_ECHO 9033
-#define n_INLINE_HTML 9034
-#define n_UNSET_LIST 9035
-#define n_UNSET 9036
-#define n_FOREACH 9037
-#define n_FOREACH_EXPRESSION 9038
-#define n_THROW 9039
-#define n_GOTO 9040
-#define n_TRY 9041
-#define n_CATCH_LIST 9042
-#define n_CATCH 9043
-#define n_DECLARE 9044
-#define n_DECLARE_DECLARATION_LIST 9045
-#define n_DECLARE_DECLARATION 9046
-#define n_VARIABLE 9047
-#define n_REFERENCE 9048
-#define n_VARIABLE_REFERENCE 9049
-#define n_FUNCTION_DECLARATION 9050
-#define n_CLASS_DECLARATION 9051
-#define n_CLASS_ATTRIBUTES 9052
-#define n_EXTENDS 9053
-#define n_EXTENDS_LIST 9054
-#define n_IMPLEMENTS_LIST 9055
-#define n_INTERFACE_DECLARATION 9056
-#define n_CASE 9057
-#define n_DEFAULT 9058
-#define n_DECLARATION_PARAMETER_LIST 9059
-#define n_DECLARATION_PARAMETER 9060
-#define n_TYPE_NAME 9061
-#define n_VARIABLE_VARIABLE 9062
-#define n_CLASS_MEMBER_DECLARATION_LIST 9063
-#define n_CLASS_MEMBER_DECLARATION 9064
-#define n_CLASS_CONSTANT_DECLARATION_LIST 9065
-#define n_CLASS_CONSTANT_DECLARATION 9066
-#define n_METHOD_DECLARATION 9067
-#define n_METHOD_MODIFIER_LIST 9068
-#define n_FUNCTION_MODIFIER_LIST 9069
-#define n_CLASS_MEMBER_MODIFIER_LIST 9070
-#define n_EXPRESSION_LIST 9071
-#define n_LIST 9072
-#define n_ASSIGNMENT 9073
-#define n_NEW 9074
-#define n_UNARY_PREFIX_EXPRESSION 9075
-#define n_UNARY_POSTFIX_EXPRESSION 9076
-#define n_BINARY_EXPRESSION 9077
-#define n_TERNARY_EXPRESSION 9078
-#define n_CAST_EXPRESSION 9079
-#define n_CAST 9080
-#define n_OPERATOR 9081
-#define n_ARRAY_LITERAL 9082
-#define n_EXIT_EXPRESSION 9083
-#define n_BACKTICKS_EXPRESSION 9084
-#define n_LEXICAL_VARIABLE_LIST 9085
-#define n_NUMERIC_SCALAR 9086
-#define n_STRING_SCALAR 9087
-#define n_MAGIC_SCALAR 9088
-#define n_CLASS_STATIC_ACCESS 9089
-#define n_CLASS_NAME 9090
-#define n_MAGIC_CLASS_KEYWORD 9091
-#define n_OBJECT_PROPERTY_ACCESS 9092
-#define n_ARRAY_VALUE_LIST 9093
-#define n_ARRAY_VALUE 9094
-#define n_CALL_PARAMETER_LIST 9095
-#define n_VARIABLE_EXPRESSION 9096
-#define n_INCLUDE_FILE 9097
-#define n_HEREDOC 9098
-#define n_FUNCTION_CALL 9099
-#define n_INDEX_ACCESS 9100
-#define n_ASSIGNMENT_LIST 9101
-#define n_METHOD_CALL 9102
-#define n_XHP_TAG 9103
-#define n_XHP_TAG_OPEN 9104
-#define n_XHP_TAG_CLOSE 9105
-#define n_XHP_TEXT 9106
-#define n_XHP_EXPRESSION 9107
-#define n_XHP_ATTRIBUTE_LIST 9108
-#define n_XHP_ATTRIBUTE 9109
-#define n_XHP_LITERAL 9110
-#define n_XHP_ATTRIBUTE_LITERAL 9111
-#define n_XHP_ATTRIBUTE_EXPRESSION 9112
-#define n_XHP_NODE_LIST 9113
-#define n_XHP_ENTITY 9114
-#define n_CONCATENATION_LIST 9115
-#define n_PARENTHETICAL_EXPRESSION 9116

support/xhpast/parser_nodes.php

@@ -1,139 +0,0 @@
-<?php
-
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-function xhp_parser_node_constants() {
-  return array(
-    9000 => 'n_PROGRAM',
-    9001 => 'n_SYMBOL_NAME',
-    9002 => 'n_HALT_COMPILER',
-    9003 => 'n_NAMESPACE',
-    9004 => 'n_STATEMENT',
-    9005 => 'n_EMPTY',
-    9006 => 'n_STATEMENT_LIST',
-    9007 => 'n_OPEN_TAG',
-    9008 => 'n_CLOSE_TAG',
-    9009 => 'n_USE_LIST',
-    9010 => 'n_USE',
-    9011 => 'n_CONSTANT_DECLARATION_LIST',
-    9012 => 'n_CONSTANT_DECLARATION',
-    9013 => 'n_STRING',
-    9014 => 'n_LABEL',
-    9015 => 'n_CONDITION_LIST',
-    9016 => 'n_CONTROL_CONDITION',
-    9017 => 'n_IF',
-    9018 => 'n_ELSEIF',
-    9019 => 'n_ELSE',
-    9020 => 'n_WHILE',
-    9021 => 'n_DO_WHILE',
-    9022 => 'n_FOR',
-    9023 => 'n_FOR_EXPRESSION',
-    9024 => 'n_SWITCH',
-    9025 => 'n_BREAK',
-    9026 => 'n_CONTINUE',
-    9027 => 'n_RETURN',
-    9028 => 'n_GLOBAL_DECLARATION_LIST',
-    9029 => 'n_GLOBAL_DECLARATION',
-    9030 => 'n_STATIC_DECLARATION_LIST',
-    9031 => 'n_STATIC_DECLARATION',
-    9032 => 'n_ECHO_LIST',
-    9033 => 'n_ECHO',
-    9034 => 'n_INLINE_HTML',
-    9035 => 'n_UNSET_LIST',
-    9036 => 'n_UNSET',
-    9037 => 'n_FOREACH',
-    9038 => 'n_FOREACH_EXPRESSION',
-    9039 => 'n_THROW',
-    9040 => 'n_GOTO',
-    9041 => 'n_TRY',
-    9042 => 'n_CATCH_LIST',
-    9043 => 'n_CATCH',
-    9044 => 'n_DECLARE',
-    9045 => 'n_DECLARE_DECLARATION_LIST',
-    9046 => 'n_DECLARE_DECLARATION',
-    9047 => 'n_VARIABLE',
-    9048 => 'n_REFERENCE',
-    9049 => 'n_VARIABLE_REFERENCE',
-    9050 => 'n_FUNCTION_DECLARATION',
-    9051 => 'n_CLASS_DECLARATION',
-    9052 => 'n_CLASS_ATTRIBUTES',
-    9053 => 'n_EXTENDS',
-    9054 => 'n_EXTENDS_LIST',
-    9055 => 'n_IMPLEMENTS_LIST',
-    9056 => 'n_INTERFACE_DECLARATION',
-    9057 => 'n_CASE',
-    9058 => 'n_DEFAULT',
-    9059 => 'n_DECLARATION_PARAMETER_LIST',
-    9060 => 'n_DECLARATION_PARAMETER',
-    9061 => 'n_TYPE_NAME',
-    9062 => 'n_VARIABLE_VARIABLE',
-    9063 => 'n_CLASS_MEMBER_DECLARATION_LIST',
-    9064 => 'n_CLASS_MEMBER_DECLARATION',
-    9065 => 'n_CLASS_CONSTANT_DECLARATION_LIST',
-    9066 => 'n_CLASS_CONSTANT_DECLARATION',
-    9067 => 'n_METHOD_DECLARATION',
-    9068 => 'n_METHOD_MODIFIER_LIST',
-    9069 => 'n_FUNCTION_MODIFIER_LIST',
-    9070 => 'n_CLASS_MEMBER_MODIFIER_LIST',
-    9071 => 'n_EXPRESSION_LIST',
-    9072 => 'n_LIST',
-    9073 => 'n_ASSIGNMENT',
-    9074 => 'n_NEW',
-    9075 => 'n_UNARY_PREFIX_EXPRESSION',
-    9076 => 'n_UNARY_POSTFIX_EXPRESSION',
-    9077 => 'n_BINARY_EXPRESSION',
-    9078 => 'n_TERNARY_EXPRESSION',
-    9079 => 'n_CAST_EXPRESSION',
-    9080 => 'n_CAST',
-    9081 => 'n_OPERATOR',
-    9082 => 'n_ARRAY_LITERAL',
-    9083 => 'n_EXIT_EXPRESSION',
-    9084 => 'n_BACKTICKS_EXPRESSION',
-    9085 => 'n_LEXICAL_VARIABLE_LIST',
-    9086 => 'n_NUMERIC_SCALAR',
-    9087 => 'n_STRING_SCALAR',
-    9088 => 'n_MAGIC_SCALAR',
-    9089 => 'n_CLASS_STATIC_ACCESS',
-    9090 => 'n_CLASS_NAME',
-    9091 => 'n_MAGIC_CLASS_KEYWORD',
-    9092 => 'n_OBJECT_PROPERTY_ACCESS',
-    9093 => 'n_ARRAY_VALUE_LIST',
-    9094 => 'n_ARRAY_VALUE',
-    9095 => 'n_CALL_PARAMETER_LIST',
-    9096 => 'n_VARIABLE_EXPRESSION',
-    9097 => 'n_INCLUDE_FILE',
-    9098 => 'n_HEREDOC',
-    9099 => 'n_FUNCTION_CALL',
-    9100 => 'n_INDEX_ACCESS',
-    9101 => 'n_ASSIGNMENT_LIST',
-    9102 => 'n_METHOD_CALL',
-    9103 => 'n_XHP_TAG',
-    9104 => 'n_XHP_TAG_OPEN',
-    9105 => 'n_XHP_TAG_CLOSE',
-    9106 => 'n_XHP_TEXT',
-    9107 => 'n_XHP_EXPRESSION',
-    9108 => 'n_XHP_ATTRIBUTE_LIST',
-    9109 => 'n_XHP_ATTRIBUTE',
-    9110 => 'n_XHP_LITERAL',
-    9111 => 'n_XHP_ATTRIBUTE_LITERAL',
-    9112 => 'n_XHP_ATTRIBUTE_EXPRESSION',
-    9113 => 'n_XHP_NODE_LIST',
-    9114 => 'n_XHP_ENTITY',
-    9115 => 'n_CONCATENATION_LIST',
-    9116 => 'n_PARENTHETICAL_EXPRESSION',
-  );
-}

support/xhpast/xhpast.cpp

@@ -1,150 +0,0 @@
-/*
- * Copyright 2011 Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "ast.hpp"
-#include <vector>
-#include <string>
-#include <iostream>
-#include <sstream>
-#include <fstream>
-using namespace std;
-
-int xhpastparse(void*, xhpast::Node **);
-
-int xhpast_process(std::string &in);
-void print_node(xhpast::Node *node);
-
-int main(int argc, char* argv[]) {
-  vector<string> files;
-  
-/*  if (argc != 2) {
-    cerr << "usage: xhpast <file.php>\n";
-    return 1;
-  }
-  */
-  
-  ifstream inputFile;
-  istream *inputStream;
-//  inputFile.open(argv[1]);
-//  inputStream = &inputFile;
-  inputStream = &cin;
-  
-  std::stringbuf sb;
-  *inputStream >> noskipws >> &sb;
-  std::string buffer = sb.str();
-  inputFile.close();
-
-  return xhpast_process(buffer);
-}
-
-int xhpast_process(std::string &in) {
-  
-  char *buffer;
-  in.reserve(in.size() + 1);
-  buffer = const_cast<char*>(in.c_str());
-  buffer[in.size() + 1] = 0; // need double NULL for scan_buffer
-  
-  void* scanner;
-  yy_extra_type extra;
-  extra.idx_expr = true;//flags.idx_expr;
-  extra.include_debug = true;//flags.include_debug;
-  extra.insert_token = 0;//flags.eval ? T_OPEN_TAG_FAKE : 0;
-  extra.short_tags = true;//flags.short_tags;
-  extra.asp_tags = false;//flags.asp_tags;
-  
-  xhpast::Node *root = NULL;
-
-  xhpastlex_init(&scanner);
-  xhpastset_extra(&extra, scanner);
-  xhpast_scan_buffer(buffer, in.size() + 2, scanner);
-#ifdef DEBUG
-  xhpdebug = 1;
-#endif
-  xhpastparse(scanner, &root);
-  xhpastlex_destroy(scanner);
-
-  if (extra.terminated) {
-    fprintf(
-      stderr,
-      "XHPAST Parse Error: %s on line %d\n",
-      extra.error.c_str(),
-      (int)extra.lineno);
-    return 1;
-  }
-  
-  printf("{");
-  printf("\"tree\":");
-  if (root) {
-    // Extend the right token for the root node to the end of the concrete
-    // token stream. This ensure all tokens appear in the tree. If we don't
-    // do this and the file ends in tokens which don't go to the parser (like
-    // comments and whitespace) they won't be represented in the tree.
-    root->r_tok = (extra.token_list.size() - 1);
-    print_node(root);
-  } else {
-    printf("null");
-  }
-  printf(",");
-  printf("\"stream\":");
-  printf("[");
-  
-  if (!extra.token_list.empty()) {
-    for (xhpast::token_list_t::iterator ii = extra.token_list.begin();;) {
-      printf("[%d, %d]", (*ii)->type, (int)(*ii)->value.length());
-      if (++ii != extra.token_list.end()) {
-        printf(",");
-      } else {
-        break;
-      }
-    }
-  }
-  printf("]");
-  printf("}\n");
-  
-  return 0;
-}
-
-void print_node(xhpast::Node *node) {
-  int l = -1;
-  int r = -1;
-  if (node->l_tok != -1) {
-    l = node->l_tok;
-  }
-
-  if (l == -1) {
-    printf("[%d]", node->type);
-  } else {
-    if (node->r_tok != -1) {
-      r = node->r_tok;
-    }
-
-    printf("[%d, %d, %d", node->type, l, r);
-    if (!node->children.empty()) {
-      printf(", [");
-      for (xhpast::node_list_t::iterator ii = node->children.begin();;) {
-        print_node(*ii);
-        if (++ii != node->children.end()) {
-          printf(",");
-        } else {
-          break;
-        }
-      }
-      printf("]");
-    }
-    printf("]");
-  }
-}
-