Provide phragment.getstate and phragment.getpatch Conduit methods

Summary:
This provides a `phragment.getstate` and a `phragment.getpatch` Conduit method.

`phragment.getstate` - This returns the current state of the fragment and all of it's children.

`phragment.getpatch` - This accepts a base path and a mapping of paths to hashes.  The mapping is for the caller to specify the current state of the files it has.  This returns a list of patches that the caller needs to apply to it's files to get to the latest version.

Test Plan:
Ran the following script in a folder which had content matching a fragment and it's children:

```
#!/bin/bash

STATE=""
for i in $(find ./ -type f); do
    HASH=$(cat $i | sha1sum | awk '{ print $1 }')
    BASE=${i:2}
    STATE="$STATE,\"$BASE\":\"$HASH\""
done
STATE=${STATE:1}
STATE="{$STATE}"

echo '{"path":"tychaia3.zip","state":'$STATE'}' | arc --conduit-uri=http://phabricator.local/ call-conduit phragment.getpatch
```

and I got:

```
{"error":null,"errorMessage":null,"response":[]}
```

I updated one of the child fragments with a new file and ran the script again (patch has been omitted due to it's size):

```
{"error":null,"errorMessage":null,"response":[{"path":"Content\/TitleFont.xnb","hash_old":"4a927d7b90582e50cdd330de9f4b59b0cc5eb5c7","hash_new":"25867504642a3a403102274c68fbb9b430c1980f","patch":"..."}]}
```

Reviewers: epriestley, #blessed_reviewers

Reviewed By: epriestley

CC: Korvin, epriestley, aran, staticshock

Maniphest Tasks: T4205

Differential Revision: https://secure.phabricator.com/D7739

src/__phutil_library_map__.php

@@ -207,6 +207,9 @@ phutil_register_library_map(array(
     'ConduitAPI_phpast_Method' => 'applications/phpast/conduit/ConduitAPI_phpast_Method.php',
     'ConduitAPI_phpast_getast_Method' => 'applications/phpast/conduit/ConduitAPI_phpast_getast_Method.php',
     'ConduitAPI_phpast_version_Method' => 'applications/phpast/conduit/ConduitAPI_phpast_version_Method.php',
+    'ConduitAPI_phragment_Method' => 'applications/phragment/conduit/ConduitAPI_phragment_Method.php',
+    'ConduitAPI_phragment_getpatch_Method' => 'applications/phragment/conduit/ConduitAPI_phragment_getpatch_Method.php',
+    'ConduitAPI_phragment_queryfragments_Method' => 'applications/phragment/conduit/ConduitAPI_phragment_queryfragments_Method.php',
     'ConduitAPI_phriction_Method' => 'applications/phriction/conduit/ConduitAPI_phriction_Method.php',
     'ConduitAPI_phriction_edit_Method' => 'applications/phriction/conduit/ConduitAPI_phriction_edit_Method.php',
     'ConduitAPI_phriction_history_Method' => 'applications/phriction/conduit/ConduitAPI_phriction_history_Method.php',
@@ -2588,6 +2591,9 @@ phutil_register_library_map(array(
     'ConduitAPI_phpast_Method' => 'ConduitAPIMethod',
     'ConduitAPI_phpast_getast_Method' => 'ConduitAPI_phpast_Method',
     'ConduitAPI_phpast_version_Method' => 'ConduitAPI_phpast_Method',
+    'ConduitAPI_phragment_Method' => 'ConduitAPIMethod',
+    'ConduitAPI_phragment_getpatch_Method' => 'ConduitAPI_phragment_Method',
+    'ConduitAPI_phragment_queryfragments_Method' => 'ConduitAPI_phragment_Method',
     'ConduitAPI_phriction_Method' => 'ConduitAPIMethod',
     'ConduitAPI_phriction_edit_Method' => 'ConduitAPI_phriction_Method',
     'ConduitAPI_phriction_history_Method' => 'ConduitAPI_phriction_Method',

src/applications/files/query/PhabricatorFileQuery.php

@@ -13,6 +13,7 @@ final class PhabricatorFileQuery
   private $transforms;
   private $dateCreatedAfter;
   private $dateCreatedBefore;
+  private $contentHashes;
 
   public function withIDs(array $ids) {
     $this->ids = $ids;
@@ -39,6 +40,11 @@ final class PhabricatorFileQuery
     return $this;
   }
 
+  public function withContentHashes(array $content_hashes) {
+    $this->contentHashes = $content_hashes;
+    return $this;
+  }
+
   /**
    * Select files which are transformations of some other file. For example,
    * you can use this query to find previously generated thumbnails of an image
@@ -228,6 +234,13 @@ final class PhabricatorFileQuery
         $this->dateCreatedBefore);
     }
 
+    if ($this->contentHashes) {
+      $where[] = qsprintf(
+        $conn_r,
+        'f.contentHash IN (%Ls)',
+        $this->contentHashes);
+    }
+
     return $this->formatWhereClause($where);
   }
 

src/applications/phragment/conduit/ConduitAPI_phragment_Method.php

@@ -0,0 +1,13 @@
+<?php
+
+/**
+ * @group conduit
+ */
+abstract class ConduitAPI_phragment_Method extends ConduitAPIMethod {
+
+  public function getApplication() {
+    return PhabricatorApplication::getByClass(
+      'PhabricatorApplicationPhragment');
+  }
+
+}

src/applications/phragment/conduit/ConduitAPI_phragment_getpatch_Method.php

@@ -0,0 +1,185 @@
+<?php
+
+/**
+ * @group conduit
+ */
+final class ConduitAPI_phragment_getpatch_Method
+  extends ConduitAPI_phragment_Method {
+
+  public function getMethodStatus() {
+    return self::METHOD_STATUS_UNSTABLE;
+  }
+
+  public function getMethodDescription() {
+    return pht("Retrieve the patches to apply for a given set of files.");
+  }
+
+  public function defineParamTypes() {
+    return array(
+      'path' => 'required string',
+      'state' => 'required dict<string, string>',
+    );
+  }
+
+  public function defineReturnType() {
+    return 'nonempty dict';
+  }
+
+  public function defineErrorTypes() {
+    return array(
+      'ERR_BAD_FRAGMENT' => 'No such fragment exists',
+    );
+  }
+
+  protected function execute(ConduitAPIRequest $request) {
+    $path = $request->getValue('path');
+    $state = $request->getValue('state');
+    // The state is an array mapping file paths to hashes.
+
+    $patches = array();
+
+    // We need to get all of the mappings (like phragment.getstate) first
+    // so that we can detect deletions and creations of files.
+    $fragment = id(new PhragmentFragmentQuery())
+      ->setViewer($request->getUser())
+      ->withPaths(array($path))
+      ->executeOne();
+    if ($fragment === null) {
+      throw new ConduitException('ERR_BAD_FRAGMENT');
+    }
+
+    $mappings = $fragment->getFragmentMappings(
+      $request->getUser(),
+      $fragment->getPath());
+
+    $file_phids = mpull(mpull($mappings, 'getLatestVersion'), 'getFilePHID');
+    $files = id(new PhabricatorFileQuery())
+      ->setViewer($request->getUser())
+      ->withPHIDs($file_phids)
+      ->execute();
+    $files = mpull($files, null, 'getPHID');
+
+    // Scan all of the files that the caller currently has and iterate
+    // over that.
+    foreach ($state as $path => $hash) {
+      // If $mappings[$path] exists, then the user has the file and it's
+      // also a fragment.
+      if (array_key_exists($path, $mappings)) {
+        $file_phid = $mappings[$path]->getLatestVersion()->getFilePHID();
+        if ($file_phid !== null) {
+          // If the file PHID is present, then we need to check the
+          // hashes to see if they are the same.
+          $hash_caller = strtolower($state[$path]);
+          $hash_current = $files[$file_phid]->getContentHash();
+          if ($hash_caller === $hash_current) {
+            // The user's version is identical to our version, so
+            // there is no update needed.
+          } else {
+            // The hash differs, and the user needs to update.
+            $patches[] = array(
+              'path' => $path,
+              'fileOld' => null,
+              'fileNew' => $files[$file_phid],
+              'hashOld' => $hash_caller,
+              'hashNew' => $hash_current,
+              'patchURI' => null);
+          }
+        } else {
+          // We have a record of this as a file, but there is no file
+          // attached to the latest version, so we consider this to be
+          // a deletion.
+          $patches[] = array(
+            'path' => $path,
+            'fileOld' => null,
+            'fileNew' => null,
+            'hashOld' => $hash_caller,
+            'hashNew' => PhragmentPatchUtil::EMPTY_HASH,
+            'patchURI' => null);
+        }
+      } else {
+        // If $mappings[$path] does not exist, then the user has a file,
+        // and we have absolutely no record of it what-so-ever (we haven't
+        // even recorded a deletion).  Assuming most applications will store
+        // some form of data near their own files, this is probably a data
+        // file relevant for the application that is not versioned, so we
+        // don't tell the client to do anything with it.
+      }
+    }
+
+    // Check the remaining files that we know about but the caller has
+    // not reported.
+    foreach ($mappings as $path => $child) {
+      if (array_key_exists($path, $state)) {
+        // We have already evaluated this above.
+      } else {
+        $file_phid = $mappings[$path]->getLatestVersion()->getFilePHID();
+        if ($file_phid !== null) {
+          // If the file PHID is present, then this is a new file that
+          // we know about, but the caller does not.  We need to tell
+          // the caller to create the file.
+          $hash_current = $files[$file_phid]->getContentHash();
+          $patches[] = array(
+            'path' => $path,
+            'fileOld' => null,
+            'fileNew' => $files[$file_phid],
+            'hashOld' => PhragmentPatchUtil::EMPTY_HASH,
+            'hashNew' => $hash_current,
+            'patchURI' => null);
+        } else {
+          // We have a record of deleting this file, and the caller hasn't
+          // reported it, so they've probably deleted it in a previous
+          // update.
+        }
+      }
+    }
+
+    // Before we can calculate patches, we need to resolve the old versions
+    // of files so we can draw diffs on them.
+    $hashes = array();
+    foreach ($patches as $patch) {
+      if ($patch["hashOld"] !== PhragmentPatchUtil::EMPTY_HASH) {
+        $hashes[] = $patch["hashOld"];
+      }
+    }
+    $old_files = array();
+    if (count($hashes) !== 0) {
+      $old_files = id(new PhabricatorFileQuery())
+        ->setViewer($request->getUser())
+        ->withContentHashes($hashes)
+        ->execute();
+    }
+    $old_files = mpull($old_files, null, 'getContentHash');
+    foreach ($patches as $key => $patch) {
+      if ($patch["hashOld"] !== PhragmentPatchUtil::EMPTY_HASH) {
+        if (array_key_exists($patch['hashOld'], $old_files)) {
+          $patches[$key]['fileOld'] = $old_files[$patch['hashOld']];
+        } else {
+          // We either can't see or can't read the old file.
+          $patches[$key]['hashOld'] = PhragmentPatchUtil::EMPTY_HASH;
+          $patches[$key]['fileOld'] = null;
+        }
+      }
+    }
+
+    // Now run through all of the patch entries, calculate the patches
+    // and return the results.
+    foreach ($patches as $key => $patch) {
+      $data = PhragmentPatchUtil::calculatePatch(
+        $patches[$key]['fileOld'],
+        $patches[$key]['fileNew']);
+      unset($patches[$key]['fileOld']);
+      unset($patches[$key]['fileNew']);
+
+      $file = PhabricatorFile::buildFromFileDataOrHash(
+        $data,
+        array(
+          'name' => 'patch.dmp',
+          'ttl' => time() + 60 * 60 * 24,
+        ));
+      $patches[$key]['patchURI'] = $file->getDownloadURI();
+    }
+
+    return $patches;
+  }
+
+}

src/applications/phragment/conduit/ConduitAPI_phragment_queryfragments_Method.php

@@ -0,0 +1,83 @@
+<?php
+
+/**
+ * @group conduit
+ */
+final class ConduitAPI_phragment_queryfragments_Method
+  extends ConduitAPI_phragment_Method {
+
+  public function getMethodStatus() {
+    return self::METHOD_STATUS_UNSTABLE;
+  }
+
+  public function getMethodDescription() {
+    return pht("Query fragments based on their paths.");
+  }
+
+  public function defineParamTypes() {
+    return array(
+      'paths' => 'required list<string>',
+    );
+  }
+
+  public function defineReturnType() {
+    return 'nonempty dict';
+  }
+
+  public function defineErrorTypes() {
+    return array(
+      'ERR_BAD_FRAGMENT' => 'No such fragment exists',
+    );
+  }
+
+  protected function execute(ConduitAPIRequest $request) {
+    $paths = $request->getValue('paths');
+
+    $fragments = id(new PhragmentFragmentQuery())
+      ->setViewer($request->getUser())
+      ->withPaths($paths)
+      ->execute();
+    $fragments = mpull($fragments, null, 'getPath');
+    foreach ($paths as $path) {
+      if (!array_key_exists($path, $fragments)) {
+        throw new ConduitException('ERR_BAD_FRAGMENT');
+      }
+    }
+
+    $results = array();
+    foreach ($fragments as $path => $fragment) {
+      $mappings = $fragment->getFragmentMappings(
+        $request->getUser(),
+        $fragment->getPath());
+
+      $file_phids = mpull(mpull($mappings, 'getLatestVersion'), 'getFilePHID');
+      $files = id(new PhabricatorFileQuery())
+        ->setViewer($request->getUser())
+        ->withPHIDs($file_phids)
+        ->execute();
+      $files = mpull($files, null, 'getPHID');
+
+      $result = array();
+      foreach ($mappings as $cpath => $child) {
+        $file_phid = $child->getLatestVersion()->getFilePHID();
+        if (!isset($files[$file_phid])) {
+          // Skip any files we don't have permission to access.
+          continue;
+        }
+
+        $file = $files[$file_phid];
+        $cpath = substr($child->getPath(), strlen($fragment->getPath()) + 1);
+        $result[] = array(
+          'phid' => $child->getPHID(),
+          'phidVersion' => $child->getLatestVersionPHID(),
+          'path' => $cpath,
+          'hash' => $file->getContentHash(),
+          'version' => $child->getLatestVersion()->getSequence(),
+          'uri' => $file->getViewURI());
+      }
+      $results[$path] = $result;
+    }
+    return $results;
+  }
+
+}

src/applications/phragment/controller/PhragmentZIPController.php

@@ -111,29 +111,18 @@ final class PhragmentZIPController extends PhragmentController {
    * Returns a list of mappings like array('some/path.txt' => 'file PHID');
    */
   private function getFragmentMappings(PhragmentFragment $current, $base_path) {
-    $children = id(new PhragmentFragmentQuery())
+    $mappings = $current->getFragmentMappings(
-      ->setViewer($this->getRequest()->getUser())
+      $this->getRequest()->getUser(),
-      ->needLatestVersion(true)
+      $base_path);
-      ->withLeadingPath($current->getPath().'/')
-      ->withDepths(array($current->getDepth() + 1))
-      ->execute();
 
-    if (count($children) === 0) {
+    $result = array();
-      $path = substr($current->getPath(), strlen($base_path) + 1);
+    foreach ($mappings as $path => $fragment) {
-      if ($this->getVersion($current) === null) {
+      $version = $this->getVersion($fragment);
-        return array();
+      if ($version !== null) {
+        $result[$path] = $version->getFilePHID();
       }
-      return array($path => $this->getVersion($current)->getFilePHID());
-    } else {
-      $mappings = array();
-      foreach ($children as $child) {
-        $child_mappings = $this->getFragmentMappings($child, $base_path);
-        foreach ($child_mappings as $key => $value) {
-          $mappings[$key] = $value;
-        }
-      }
-      return $mappings;
     }
+    return $result;
   }
 
   private function getVersion($fragment) {

src/applications/phragment/storage/PhragmentFragment.php

@@ -276,6 +276,38 @@ final class PhragmentFragment extends PhragmentDAO
   }
 
 
+/* -(  Utility  )  ---------------------------------------------------------- */
+
+
+  public function getFragmentMappings(
+    PhabricatorUser $viewer,
+    $base_path) {
+
+    $children = id(new PhragmentFragmentQuery())
+      ->setViewer($viewer)
+      ->needLatestVersion(true)
+      ->withLeadingPath($this->getPath().'/')
+      ->withDepths(array($this->getDepth() + 1))
+      ->execute();
+
+    if (count($children) === 0) {
+      $path = substr($this->getPath(), strlen($base_path) + 1);
+      return array($path => $this);
+    } else {
+      $mappings = array();
+      foreach ($children as $child) {
+        $child_mappings = $child->getFragmentMappings(
+          $viewer,
+          $base_path);
+        foreach ($child_mappings as $key => $value) {
+          $mappings[$key] = $value;
+        }
+      }
+      return $mappings;
+    }
+  }
+
+
 /* -(  Policy Interface  )--------------------------------------------------- */