Make XHPAST unit test "expect" blocks stable and human-readable

Summary:
Depends on D21064. Ref T13492. Earlier, see D17819. This is essentially the same change, although I inlined the token stream into the node list.

This intentionally breaks most tests since it just has the new "expect" generator; the next change will fix them by swapping the test bodies.

Test Plan: Ran "arc unit --everything" after the next change (which fixes all the tests), got a clean build. This change on its own fails all existing XHPAST tests since the block formats don't match.

Maniphest Tasks: T13492

Differential Revision: https://secure.phabricator.com/D21065

src/parser/xhpast/__tests__/PHPASTParserTestCase.php

@@ -80,7 +80,6 @@ final class PHPASTParserTestCase extends PhutilTestCase {
 
     switch ($type) {
       case 'pass':
-      case 'fail-parse':
         $this->assertEqual(0, $err, pht('Exit code for "%s".', $name));
 
         if (!strlen($expect)) {
@@ -88,16 +87,6 @@ final class PHPASTParserTestCase extends PhutilTestCase {
           break;
         }
 
-        try {
-          $expect = phutil_json_decode($expect);
-        } catch (PhutilJSONParserException $ex) {
-          throw new PhutilProxyException(
-            pht(
-              'Expect data for test "%s" is not valid JSON.',
-              $name),
-            $ex);
-        }
-
         try {
           $stdout = phutil_json_decode($stdout);
         } catch (PhutilJSONParserException $ex) {
@@ -108,21 +97,12 @@ final class PHPASTParserTestCase extends PhutilTestCase {
             $ex);
         }
 
-        $json = new PhutilJSON();
+        $stdout_nice = $this->newReadableAST($stdout, $data);
 
-        $expect_nice = $json->encodeFormatted($expect);
+        $this->assertEqual(
-        $stdout_nice = $json->encodeFormatted($stdout);
+          $expect,
-
+          $stdout_nice,
-        if ($type == 'pass') {
+          pht('Parser output for "%s".', $name));
-          $this->assertEqual(
-            $expect_nice,
-            $stdout_nice,
-            pht('Parser output for "%s".', $name));
-        } else {
-          $this->assertFalse(
-            ($expect_nice == $stdout_nice),
-            pht('Expected parser to parse "%s" incorrectly.', $name));
-        }
         break;
       case 'fail-syntax':
         $this->assertEqual(1, $err, pht('Exit code for "%s".', $name));
@@ -130,7 +110,221 @@ final class PHPASTParserTestCase extends PhutilTestCase {
           (bool)preg_match('/syntax error/', $stderr),
           pht('Expect "syntax error" in stderr or "%s".', $name));
         break;
+      default:
+        throw new Exception(
+          pht(
+            'Unknown PHPAST parser test type "%s"!',
+            $type));
     }
   }
 
+  /**
+   * Build a human-readable, stable, relatively diff-able string representing
+   * an AST (both the node tree itself and the accompanying token stream) for
+   * use in unit tests.
+   */
+  private function newReadableAST(array $data, $source) {
+    $tree = new XHPASTTree($data['tree'], $data['stream'], $source);
+
+    $root = $tree->getRootNode();
+
+    $depth = 0;
+    $list = $this->newReadableTreeLines($root, $depth);
+
+    return implode('', $list);
+  }
+
+  private function newReadableTreeLines(AASTNode $node, $depth) {
+    $out = array();
+
+    try {
+      $type_name = $node->getTypeName();
+    } catch (Exception $ex) {
+      $type_name = sprintf('<INVALID TYPE "%s">', $node->getTypeID());
+    }
+
+    $out[] = $this->newBlock($depth, '*', $type_name);
+
+    $tokens = $node->getTokens();
+
+    if ($tokens) {
+      $l = head_key($tokens);
+      $r = last_key($tokens);
+    } else {
+      $l = null;
+      $r = null;
+    }
+
+    $items = array();
+
+    $child_token_map = array();
+
+    $children = $node->getChildren();
+    foreach ($children as $child) {
+      $child_tokens = $child->getTokens();
+
+      if ($child_tokens) {
+        $child_l = head_key($child_tokens);
+        $child_r = last_key($child_tokens);
+      } else {
+        $child_l = null;
+        $child_r = null;
+      }
+
+      if ($l !== null) {
+        for ($ii = $l; $ii < $child_l; $ii++) {
+          $items[] = $tokens[$ii];
+        }
+      }
+
+      $items[] = $child;
+
+      if ($child_r !== null) {
+        // NOTE: In some cases, child nodes do not appear in token order.
+        // That is, the 4th child of a node may use tokens that appear
+        // between children 2 and 3. Ideally, we wouldn't have cases of
+        // this and wouldn't have a positional AST.
+
+        // Work around this by: never moving the token cursor backwards; and
+        // explicitly preventing tokens appearing in any child from being
+        // printed at top level.
+
+        for ($ii = $child_l; $ii <= $child_r; $ii++) {
+          if (!isset($tokens[$ii])) {
+            continue;
+          }
+          $child_token_map[$tokens[$ii]->getTokenID()] = true;
+        }
+
+        $l = max($l, $child_r + 1);
+      } else {
+        $l = null;
+      }
+    }
+
+    if ($l !== null) {
+      for ($ii = $l; $ii <= $r; $ii++) {
+        $items[] = $tokens[$ii];
+      }
+    }
+
+    // See above. If we have tokens in the list which are part of a
+    // child node that appears later, remove them now.
+    foreach ($items as $key => $item) {
+      if ($item instanceof AASTToken) {
+        $token = $item;
+        $token_id = $token->getTokenID();
+
+        if (isset($child_token_map[$token_id])) {
+          unset($items[$key]);
+        }
+      }
+    }
+
+    foreach ($items as $item) {
+      if ($item instanceof AASTNode) {
+        $lines = $this->newReadableTreeLines($item, $depth + 1);
+        foreach ($lines as $line) {
+          $out[] = $line;
+        }
+      } else {
+        $token_value = $item->getValue();
+
+        $out[] = $this->newBlock($depth + 1, '>', $token_value);
+      }
+    }
+
+    return $out;
+  }
+
+  private function newBlock($depth, $type, $text) {
+    $output_width = 80;
+    $usable_width = ($output_width - $depth - 2);
+
+    $must_escape = false;
+
+    // We must escape the text if it isn't just simple printable characters.
+    if (preg_match('/[ \\\\\\r\\n\\t\\"]/', $text)) {
+      $must_escape = true;
+    }
+
+    // We must escape the text if it has trailing whitespace.
+    if (preg_match('/ \z/', $text)) {
+      $must_escape = true;
+    }
+
+    // We must escape the text if it won't fit on a single line.
+    if (strlen($text) > $usable_width) {
+      $must_escape = true;
+    }
+
+    if (!$must_escape) {
+      $lines = array($text);
+    } else {
+      $vector = phutil_utf8v_combined($text);
+
+      $escape_map = array(
+        "\r" => '\\r',
+        "\n" => '\\n',
+        "\t" => '\\t',
+        '"' => '\\"',
+        '\\' => '\\',
+      );
+
+      $escaped = array();
+      foreach ($vector as $key => $word) {
+        if (isset($escape_map[$word])) {
+          $vector[$key] = $escape_map[$word];
+        }
+      }
+
+
+      $line_l = '"';
+      $line_r = '"';
+
+      $max_width = ($usable_width - strlen($line_l) - strlen($line_r));
+
+      $line = '';
+      $len = 0;
+
+      $lines = array();
+      foreach ($vector as $word) {
+        $word_length = phutil_utf8_console_strlen($word);
+
+        if ($len + $word_length > $max_width) {
+          $lines[] = $line_l.$line.$line_r;
+
+          $line = '';
+          $len = 0;
+        }
+
+        $line .= $word;
+        $len += $word_length;
+      }
+
+      $lines[] = $line_l.$line.$line_r;
+    }
+
+    $is_first = true;
+    $indent = str_repeat(' ', $depth);
+
+    $output = array();
+    foreach ($lines as $line) {
+      if ($is_first) {
+        $marker = $type;
+        $is_first = false;
+      } else {
+        $marker = '.';
+      }
+
+      $output[] = sprintf(
+        "%s%s %s\n",
+        $indent,
+        $marker,
+        $line);
+    }
+
+    return implode('', $output);
+  }
+
 }

src/parser/xhpast/__tests__/data/a-self-test.php.test

@@ -0,0 +1,45 @@
+<?php
+
+$the_senate = <<<EOTRAGEDY
+Did you ever hear the tragedy of Darth Plagueis The Wise? I
+thought not. It's not a story the Jedi would tell you. It's a Sith legend.
+Darth Plagueis was a Dark Lord of the Sith, so powerful and so wise he could
+use the Force to influence the midichlorians to create life... He had such a
+knowledge of the dark side that he could even keep the ones he cared about from
+dying. The dark side of the Force is a pathway to many abilities some consider
+to be unnatural. He became so powerful... the only thing he was afraid of was
+losing his power, which eventually, of course, he did. Unfortunately, he taught
+his apprentice everything he knew, then his apprentice killed him in his sleep.
+Ironic. He could save others from death, but not himself.
+EOTRAGEDY;
+
+~~~~~~~~~~
+pass
+~~~~~~~~~~
+* n_PROGRAM
+ * n_STATEMENT_LIST
+  * n_OPEN_TAG
+   > <?php
+  > "\n\n"
+  * n_STATEMENT
+   * n_BINARY_EXPRESSION
+    * n_VARIABLE
+     > $the_senate
+    > " "
+    * n_OPERATOR
+     > =
+    > " "
+    * n_HEREDOC
+     > "<<<EOTRAGEDY\nDid you ever hear the tragedy of Darth Plagueis The Wise?"
+     . " I\nthought not. It's not a story the Jedi would tell you. It's a Sith "
+     . "legend.\nDarth Plagueis was a Dark Lord of the Sith, so powerful and so"
+     . " wise he could\nuse the Force to influence the midichlorians to create "
+     . "life... He had such a\nknowledge of the dark side that he could even ke"
+     . "ep the ones he cared about from\ndying. The dark side of the Force is a"
+     . " pathway to many abilities some consider\nto be unnatural. He became so"
+     . " powerful... the only thing he was afraid of was\nlosing his power, whi"
+     . "ch eventually, of course, he did. Unfortunately, he taught\nhis apprent"
+     . "ice everything he knew, then his apprentice killed him in his sleep.\nI"
+     . "ronic. He could save others from death, but not himself.\nEOTRAGEDY"
+   > ;
+ > "\n\n"

src/parser/xhpast/__tests__/data/base-fail-parse.php.test

@@ -1,8 +0,0 @@
-<?php
-~~~~~~~~~~
-fail-parse
-~~~~~~~~~~
-{
-  "tree": [],
-  "stream": []
-}

src/parser/xhpast/__tests__/data/php-heredoc-terminal.php.test

@@ -1,53 +0,0 @@
-<?php
-<<<HEREDOC
-HEREDOC;
-~~~~~~~~~~
-fail-parse, rtrim
-~~~~~~~~~~
-{
-  "tree": [
-    9000,
-    0,
-    2,
-    [
-      [
-        9006,
-        0,
-        2,
-        [
-          [
-            9007,
-            0,
-            0
-          ],
-          [
-            9004,
-            1,
-            2,
-            [
-              [
-                9098,
-                1,
-                1
-              ]
-            ]
-          ]
-        ]
-      ]
-    ]
-  ],
-  "stream": [
-    [
-      371,
-      6
-    ],
-    [
-      378,
-      18
-    ],
-    [
-      59,
-      1
-    ]
-  ]
-}