Support evaluation of complex tokenizer functions

Summary:
Depends on D19088. Ref T13079.

> Any sufficiently complicated C or Fortran program contains an ad hoc, informally-specified, bug-ridden, slow implementation of half of Common Lisp.
> - Greenspun's Tenth Rule

Move us a step closer to this noble goal.

This doesn't implement any `viewer(project())` stuff but it looks like the API doesn't need to change to do that in the future.

Test Plan: Grimmaced in pain.

Maniphest Tasks: T13079

Differential Revision: https://secure.phabricator.com/D19089

src/__phutil_library_map__.php

@@ -4381,6 +4381,7 @@ phutil_register_library_map(array(
     'PhabricatorTypeaheadMonogramDatasource' => 'applications/typeahead/datasource/PhabricatorTypeaheadMonogramDatasource.php',
     'PhabricatorTypeaheadResult' => 'applications/typeahead/storage/PhabricatorTypeaheadResult.php',
     'PhabricatorTypeaheadRuntimeCompositeDatasource' => 'applications/typeahead/datasource/PhabricatorTypeaheadRuntimeCompositeDatasource.php',
+    'PhabricatorTypeaheadTestNumbersDatasource' => 'applications/typeahead/datasource/__tests__/PhabricatorTypeaheadTestNumbersDatasource.php',
     'PhabricatorTypeaheadTokenView' => 'applications/typeahead/view/PhabricatorTypeaheadTokenView.php',
     'PhabricatorUIConfigOptions' => 'applications/config/option/PhabricatorUIConfigOptions.php',
     'PhabricatorUIExample' => 'applications/uiexample/examples/PhabricatorUIExample.php',
@@ -10159,6 +10160,7 @@ phutil_register_library_map(array(
     'PhabricatorTypeaheadMonogramDatasource' => 'PhabricatorTypeaheadDatasource',
     'PhabricatorTypeaheadResult' => 'Phobject',
     'PhabricatorTypeaheadRuntimeCompositeDatasource' => 'PhabricatorTypeaheadCompositeDatasource',
+    'PhabricatorTypeaheadTestNumbersDatasource' => 'PhabricatorTypeaheadDatasource',
     'PhabricatorTypeaheadTokenView' => 'AphrontTagView',
     'PhabricatorUIConfigOptions' => 'PhabricatorApplicationConfigOptions',
     'PhabricatorUIExample' => 'Phobject',

src/applications/typeahead/datasource/PhabricatorTypeaheadDatasource.php

@@ -406,7 +406,7 @@ abstract class PhabricatorTypeaheadDatasource extends Phobject {
     $results = $this->evaluateValues($results);
 
     foreach ($evaluate as $result_key => $function) {
-      $function = self::parseFunction($function);
+      $function = $this->parseFunction($function);
       if (!$function) {
         throw new PhabricatorTypeaheadInvalidTokenException();
       }
@@ -459,28 +459,69 @@ abstract class PhabricatorTypeaheadDatasource extends Phobject {
   /**
    * @task functions
    */
-  public function parseFunction($token, $allow_partial = false) {
+  protected function parseFunction($token, $allow_partial = false) {
     $matches = null;
 
     if ($allow_partial) {
-      $ok = preg_match('/^([^(]+)\((.*?)\)?$/', $token, $matches);
+      $ok = preg_match('/^([^(]+)\((.*?)\)?\z/', $token, $matches);
     } else {
-      $ok = preg_match('/^([^(]+)\((.*)\)$/', $token, $matches);
+      $ok = preg_match('/^([^(]+)\((.*)\)\z/', $token, $matches);
     }
 
     if (!$ok) {
+      if (!$allow_partial) {
+        throw new PhabricatorTypeaheadInvalidTokenException(
+          pht(
+            'Unable to parse function and arguments for token "%s".',
+            $token));
+      }
       return null;
     }
 
     $function = trim($matches[1]);
 
     if (!$this->canEvaluateFunction($function)) {
+      if (!$allow_partial) {
+        throw new PhabricatorTypeaheadInvalidTokenException(
+          pht(
+            'This datasource ("%s") can not evaluate the function "%s(...)".',
+            get_class($this),
+            $function));
+      }
+
       return null;
     }
 
+    // TODO: There is currently no way to quote characters in arguments, so
+    // some characters can't be argument characters. Replace this with a real
+    // parser once we get use cases.
+
+    $argv = $matches[2];
+    $argv = trim($argv);
+    if (!strlen($argv)) {
+      $argv = array();
+    } else {
+      $argv = preg_split('/,/', $matches[2]);
+      foreach ($argv as $key => $arg) {
+        $argv[$key] = trim($arg);
+      }
+    }
+
+    foreach ($argv as $key => $arg) {
+      if (self::isFunctionToken($arg)) {
+        $subfunction = $this->parseFunction($arg);
+
+        $results = $this->evaluateFunction(
+          $subfunction['name'],
+          array($subfunction['argv']));
+
+        $argv[$key] = head($results);
+      }
+    }
+
     return array(
       'name' => $function,
-      'argv' => array(trim($matches[2])),
+      'argv' => $argv,
     );
   }
 

src/applications/typeahead/datasource/__tests__/PhabricatorTypeaheadDatasourceTestCase.php

@@ -48,4 +48,46 @@ final class PhabricatorTypeaheadDatasourceTestCase
       pht('Tokenization of "%s"', $input));
   }
 
+  public function testFunctionEvaluation() {
+    $viewer = PhabricatorUser::getOmnipotentUser();
+
+    $datasource = id(new PhabricatorTypeaheadTestNumbersDatasource())
+      ->setViewer($viewer);
+
+    $constraints = $datasource->evaluateTokens(
+      array(
+        9,
+        'seven()',
+        12,
+        3,
+      ));
+
+    $this->assertEqual(
+      array(9, 7, 12, 3),
+      $constraints);
+
+    $map = array(
+      'inc(3)' => 4,
+      'sum(3, 4)' => 7,
+      'inc(seven())' => 8,
+      'inc(inc(3))' => 5,
+      'inc(inc(seven()))' => 9,
+      'sum(seven(), seven())' => 14,
+      'sum(inc(seven()), inc(inc(9)))' => 19,
+    );
+
+    foreach ($map as $input => $expect) {
+      $constraints = $datasource->evaluateTokens(
+        array(
+          $input,
+        ));
+
+      $this->assertEqual(
+        array($expect),
+        $constraints,
+        pht('Constraints for input "%s".', $input));
+    }
+  }
+
+
 }

src/applications/typeahead/datasource/__tests__/PhabricatorTypeaheadTestNumbersDatasource.php

@@ -0,0 +1,67 @@
+<?php
+
+final class PhabricatorTypeaheadTestNumbersDatasource
+  extends PhabricatorTypeaheadDatasource {
+
+  public function getBrowseTitle() {
+    return pht('Browse Numbers');
+  }
+
+  public function getPlaceholderText() {
+    return null;
+  }
+
+  public function getDatasourceApplicationClass() {
+    return 'PhabricatorPeopleApplication';
+  }
+
+  public function getDatasourceFunctions() {
+    return array(
+      'seven' => array(),
+      'inc' => array(),
+      'sum' => array(),
+    );
+  }
+
+  public function loadResults() {
+    return array();
+  }
+
+  public function renderFunctionTokens($function, array $argv_list) {
+    return array();
+  }
+
+  protected function evaluateFunction($function, array $argv_list) {
+    $results = array();
+
+    foreach ($argv_list as $argv) {
+      foreach ($argv as $k => $arg) {
+        if (!is_scalar($arg) || !preg_match('/^\d+\z/', $arg)) {
+          throw new PhabricatorTypeaheadInvalidTokenException(
+            pht(
+              'All arguments to "%s(...)" must be integers, found '.
+              '"%s" in position %d.',
+              $function,
+              (is_scalar($arg) ? $arg : gettype($arg)),
+              $k + 1));
+        }
+        $argv[$k] = (int)$arg;
+      }
+
+      switch ($function) {
+        case 'seven':
+          $results[] = 7;
+          break;
+        case 'inc':
+          $results[] = $argv[0] + 1;
+          break;
+        case 'sum':
+          $results[] = array_sum($argv);
+          break;
+      }
+    }
+
+    return $results;
+  }
+
+}