Implement support for creating and updating fragments from ZIPs

Summary: This implements support for creating and updating fragments from ZIP files. It allows you to upload a ZIP via the Files application, create a fragment from it, and have it recursively imported into Phragment. Updating that folder with another ZIP will recursively create, update and delete files as appropriate. The logic for creating and updating fragments from files has also been centralized into the PhragmentFragment class. Directories are also now supported; a directory fragment is simply a fragment that has no patches; thus a directory fragment can be converted to a file fragment by uploading a first patch for it. Test Plan: Uploaded ZIP files through the interface and saw all of the fragments get created and updated as expected. Reviewers: epriestley, #blessed_reviewers Reviewed By: epriestley CC: Korvin, epriestley, aran Maniphest Tasks: T4205 Differential Revision: https://secure.phabricator.com/D7729
2025-01-27 15:08:20 +01:00 · 2013-12-07 13:37:18 +11:00 · 2013-12-07 13:37:18 +11:00 · 25e7b7d53c
commit 25e7b7d53c
parent ccd4ae5638
8 changed files with 270 additions and 55 deletions
--- a/resources/sql/patches/20131206.phragmentnull.sql
+++ b/resources/sql/patches/20131206.phragmentnull.sql
@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_phragment.phragment_fragment
+MODIFY latestVersionPHID VARCHAR(64) NULL;
--- a/src/applications/phragment/controller/PhragmentBrowseController.php
+++ b/src/applications/phragment/controller/PhragmentBrowseController.php
@ -57,14 +57,18 @@ final class PhragmentBrowseController extends PhragmentController {
      $item = id(new PHUIObjectItemView());
      $item->setHeader($fragment->getName());
      $item->setHref($this->getApplicationURI('/browse/'.$fragment->getPath()));
-      $item->addAttribute(pht(
-        'Last Updated %s',
-        phabricator_datetime(
-          $fragment->getLatestVersion()->getDateCreated(),
-          $viewer)));
-      $item->addAttribute(pht(
-        'Latest Version %s',
-        $fragment->getLatestVersion()->getSequence()));
+      if (!$fragment->isDirectory()) {
+        $item->addAttribute(pht(
+          'Last Updated %s',
+          phabricator_datetime(
+            $fragment->getLatestVersion()->getDateCreated(),
+            $viewer)));
+        $item->addAttribute(pht(
+          'Latest Version %s',
+          $fragment->getLatestVersion()->getSequence()));
+      } else {
+        $item->addAttribute('Directory');
+      }
      $list->addItem($item);
    }

--- a/src/applications/phragment/controller/PhragmentController.php
+++ b/src/applications/phragment/controller/PhragmentController.php
@ -66,13 +66,16 @@ abstract class PhragmentController extends PhabricatorController {

    $this->loadHandles($phids);

-    $file = id(new PhabricatorFileQuery())
-      ->setViewer($viewer)
-      ->withPHIDs(array($fragment->getLatestVersion()->getFilePHID()))
-      ->executeOne();
+    $file = null;
    $file_uri = null;
-    if ($file !== null) {
-      $file_uri = $file->getBestURI();
+    if (!$fragment->isDirectory()) {
+      $file = id(new PhabricatorFileQuery())
+        ->setViewer($viewer)
+        ->withPHIDs(array($fragment->getLatestVersion()->getFilePHID()))
+        ->executeOne();
+      if ($file !== null) {
+        $file_uri = $file->getBestURI();
+      }
    }

    $header = id(new PHUIHeaderView())
@ -96,12 +99,21 @@ abstract class PhragmentController extends PhabricatorController {
        ->setHref($this->getApplicationURI("zip/".$fragment->getPath()))
        ->setDisabled(false) // TODO: Policy
        ->setIcon('zip'));
-    $actions->addAction(
-      id(new PhabricatorActionView())
-        ->setName(pht('Update Fragment'))
-        ->setHref($this->getApplicationURI("update/".$fragment->getPath()))
-        ->setDisabled(false) // TODO: Policy
-        ->setIcon('edit'));
+    if (!$fragment->isDirectory()) {
+      $actions->addAction(
+        id(new PhabricatorActionView())
+          ->setName(pht('Update Fragment'))
+          ->setHref($this->getApplicationURI("update/".$fragment->getPath()))
+          ->setDisabled(false) // TODO: Policy
+          ->setIcon('edit'));
+    } else {
+      $actions->addAction(
+        id(new PhabricatorActionView())
+          ->setName(pht('Convert to File'))
+          ->setHref($this->getApplicationURI("update/".$fragment->getPath()))
+          ->setDisabled(false) // TODO: Policy
+          ->setIcon('edit'));
+    }
    if ($is_history_view) {
      $actions->addAction(
        id(new PhabricatorActionView())
@ -121,9 +133,18 @@ abstract class PhragmentController extends PhabricatorController {
      ->setObject($fragment)
      ->setActionList($actions);

-    $properties->addProperty(
-      pht('Latest Version'),
-      $this->renderHandlesForPHIDs(array($fragment->getLatestVersionPHID())));
+    if (!$fragment->isDirectory()) {
+      $properties->addProperty(
+        pht('Type'),
+        pht('File'));
+      $properties->addProperty(
+        pht('Latest Version'),
+        $this->renderHandlesForPHIDs(array($fragment->getLatestVersionPHID())));
+    } else {
+      $properties->addProperty(
+        pht('Type'),
+        pht('Directory'));
+    }

    return id(new PHUIObjectBoxView())
      ->setHeader($header)
--- a/src/applications/phragment/controller/PhragmentCreateController.php
+++ b/src/applications/phragment/controller/PhragmentCreateController.php
@ -54,21 +54,12 @@ final class PhragmentCreateController extends PhragmentController {
          $depth = $parent->getDepth() + 1;
        }

-        $version = id(new PhragmentFragmentVersion());
-        $version->setSequence(0);
-        $version->setFragmentPHID(''); // Can't set this yet...
-        $version->setFilePHID($file->getPHID());
-        $version->save();
-
-        $fragment->setPath(trim($parent_path.'/'.$v_name, '/'));
-        $fragment->setDepth($depth);
-        $fragment->setLatestVersionPHID($version->getPHID());
-        $fragment->setViewPolicy($v_viewpolicy);
-        $fragment->setEditPolicy($v_editpolicy);
-        $fragment->save();
-
-        $version->setFragmentPHID($fragment->getPHID());
-        $version->save();
+        PhragmentFragment::createFromFile(
+          $viewer,
+          $file,
+          trim($parent_path.'/'.$v_name, '/'),
+          $v_viewpolicy,
+          $v_editpolicy);

        return id(new AphrontRedirectResponse())
          ->setURI('/phragment/browse/'.trim($parent_path.'/'.$v_name, '/'));
--- a/src/applications/phragment/controller/PhragmentUpdateController.php
+++ b/src/applications/phragment/controller/PhragmentUpdateController.php
@ -31,22 +31,14 @@ final class PhragmentUpdateController extends PhragmentController {
      }

      if (!count($errors)) {
-        $existing = id(new PhragmentFragmentVersionQuery())
-          ->setViewer($viewer)
-          ->withFragmentPHIDs(array($fragment->getPHID()))
-          ->execute();
-        $sequence = count($existing);
-
-        $fragment->openTransaction();
-          $version = id(new PhragmentFragmentVersion());
-          $version->setSequence($sequence);
-          $version->setFragmentPHID($fragment->getPHID());
-          $version->setFilePHID($file->getPHID());
-          $version->save();
-
-          $fragment->setLatestVersionPHID($version->getPHID());
-          $fragment->save();
-        $fragment->saveTransaction();
+        // If the file is a ZIP archive (has application/zip mimetype)
+        // then we extract the zip and apply versions for each of the
+        // individual fragments, creating and deleting files as needed.
+        if ($file->getMimeType() === "application/zip") {
+          $fragment->updateFromZIP($viewer, $file);
+        } else {
+          $fragment->updateFromFile($viewer, $file);
+        }

        return id(new AphrontRedirectResponse())
          ->setURI('/phragment/browse/'.$fragment->getPath());
--- a/src/applications/phragment/query/PhragmentFragmentQuery.php
+++ b/src/applications/phragment/query/PhragmentFragmentQuery.php
@ -115,7 +115,6 @@ final class PhragmentFragmentQuery
      foreach ($page as $key => $fragment) {
        $version_phid = $fragment->getLatestVersionPHID();
        if (empty($versions[$version_phid])) {
-          unset($page[$key]);
          continue;
        }
        $fragment->attachLatestVersion($versions[$version_phid]);
--- a/src/applications/phragment/storage/PhragmentFragment.php
+++ b/src/applications/phragment/storage/PhragmentFragment.php
@ -38,7 +38,14 @@ final class PhragmentFragment extends PhragmentDAO
    return $this->file = $file;
  }

+  public function isDirectory() {
+    return $this->latestVersionPHID === null;
+  }
+
  public function getLatestVersion() {
+    if ($this->latestVersionPHID === null) {
+      return null;
+    }
    return $this->assertAttached($this->latestVersion);
  }

@ -46,6 +53,201 @@ final class PhragmentFragment extends PhragmentDAO
    return $this->latestVersion = $version;
  }

+
+/* -(  Updating  )  --------------------------------------------------------- */
+
+
+  /**
+   * Create a new fragment from a file.
+   */
+  public static function createFromFile(
+    PhabricatorUser $viewer,
+    PhabricatorFile $file = null,
+    $path,
+    $view_policy,
+    $edit_policy) {
+
+    $fragment = id(new PhragmentFragment());
+    $fragment->setPath($path);
+    $fragment->setDepth(count(explode('/', $path)));
+    $fragment->setLatestVersionPHID(null);
+    $fragment->setViewPolicy($view_policy);
+    $fragment->setEditPolicy($edit_policy);
+    $fragment->save();
+
+    // Directory fragments have no versions associated with them, so we
+    // just return the fragment at this point.
+    if ($file === null) {
+      return $fragment;
+    }
+
+    if ($file->getMimeType() === "application/zip") {
+      $fragment->updateFromZIP($viewer, $file);
+    } else {
+      $fragment->updateFromFile($viewer, $file);
+    }
+
+    return $fragment;
+  }
+
+
+  /**
+   * Set the specified file as the next version for the fragment.
+   */
+  public function updateFromFile(
+    PhabricatorUser $viewer,
+    PhabricatorFile $file) {
+
+    $existing = id(new PhragmentFragmentVersionQuery())
+      ->setViewer($viewer)
+      ->withFragmentPHIDs(array($this->getPHID()))
+      ->execute();
+    $sequence = count($existing);
+
+    $this->openTransaction();
+      $version = id(new PhragmentFragmentVersion());
+      $version->setSequence($sequence);
+      $version->setFragmentPHID($this->getPHID());
+      $version->setFilePHID($file->getPHID());
+      $version->save();
+
+      $this->setLatestVersionPHID($version->getPHID());
+      $this->save();
+    $this->saveTransaction();
+  }
+
+  /**
+   * Apply the specified ZIP archive onto the fragment, removing
+   * and creating fragments as needed.
+   */
+  public function updateFromZIP(
+    PhabricatorUser $viewer,
+    PhabricatorFile $file) {
+
+    if ($file->getMimeType() !== "application/zip") {
+      throw new Exception("File must have mimetype 'application/zip'");
+    }
+
+    // First apply the ZIP as normal.
+    $this->updateFromFile($viewer, $file);
+
+    // Ensure we have ZIP support.
+    $zip = null;
+    try {
+      $zip = new ZipArchive();
+    } catch (Exception $e) {
+      // The server doesn't have php5-zip, so we can't do recursive updates.
+      return;
+    }
+
+    $temp = new TempFile();
+    Filesystem::writeFile($temp, $file->loadFileData());
+    if (!$zip->open($temp)) {
+      throw new Exception("Unable to open ZIP");
+    }
+
+    // Get all of the paths and their data from the ZIP.
+    $mappings = array();
+    for ($i = 0; $i < $zip->numFiles; $i++) {
+      $path = trim($zip->getNameIndex($i), '/');
+      $stream = $zip->getStream($path);
+      $data = null;
+      // If the stream is false, then it is a directory entry.  We leave
+      // $data set to null for directories so we know not to create a
+      // version entry for them.
+      if ($stream !== false) {
+        $data = stream_get_contents($stream);
+        fclose($stream);
+      }
+      $mappings[$path] = $data;
+    }
+
+    // Adjust the paths relative to this fragment so we can look existing
+    // fragments up in the DB.
+    $base_path = $this->getPath();
+    $paths = array();
+    foreach ($mappings as $p => $data) {
+      $paths[] = $base_path.'/'.$p;
+    }
+
+    // FIXME: What happens when a child exists, but the current user
+    // can't see it.  We're going to create a new child with the exact
+    // same path and then bad things will happen.
+    $children = id(new PhragmentFragmentQuery())
+      ->setViewer($viewer)
+      ->needLatestVersion(true)
+      ->withPaths($paths)
+      ->execute();
+    $children = mpull($children, null, 'getPath');
+
+    // Iterate over the existing fragments.
+    foreach ($children as $full_path => $child) {
+      $path = substr($full_path, strlen($base_path) + 1);
+      if (array_key_exists($path, $mappings)) {
+        if ($child->isDirectory() && $mappings[$path] === null) {
+          // Don't create a version entry for a directory
+          // (unless it's been converted into a file).
+          continue;
+        }
+
+        // The file is being updated.
+        $file = PhabricatorFile::newFromFileData(
+          $mappings[$path],
+          array('name' => basename($path)));
+        $child->updateFromFile($viewer, $file);
+      } else {
+        // The file is being deleted.
+        $child->deleteFile($viewer);
+      }
+    }
+
+    // Iterate over the mappings to find new files.
+    foreach ($mappings as $path => $data) {
+      if (!array_key_exists($base_path.'/'.$path, $children)) {
+        // The file is being created.  If the data is null,
+        // then this is explicitly a directory being created.
+        $file = null;
+        if ($mappings[$path] !== null) {
+          $file = PhabricatorFile::newFromFileData(
+            $mappings[$path],
+            array('name' => basename($path)));
+        }
+        PhragmentFragment::createFromFile(
+          $viewer,
+          $file,
+          $base_path.'/'.$path,
+          $this->getViewPolicy(),
+          $this->getEditPolicy());
+      }
+    }
+  }
+
+  /**
+   * Delete the contents of the specified fragment.
+   */
+  public function deleteFile(PhabricatorUser $viewer) {
+    $existing = id(new PhragmentFragmentVersionQuery())
+      ->setViewer($viewer)
+      ->withFragmentPHIDs(array($this->getPHID()))
+      ->execute();
+    $sequence = count($existing);
+
+    $this->openTransaction();
+      $version = id(new PhragmentFragmentVersion());
+      $version->setSequence($sequence);
+      $version->setFragmentPHID($this->getPHID());
+      $version->setFilePHID(null);
+      $version->save();
+
+      $this->setLatestVersionPHID($version->getPHID());
+      $this->save();
+    $this->saveTransaction();
+  }
+
+
+/* -(  Policy Interface  )--------------------------------------------------- */
+
+
  public function getCapabilities() {
    return array(
      PhabricatorPolicyCapability::CAN_VIEW,
--- a/src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php
+++ b/src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php
@ -1820,6 +1820,10 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList {
        'type' => 'sql',
        'name' => $this->getPatchPath('20131206.phragment.sql'),
      ),
+      '20131206.phragmentnull.sql' => array(
+        'type' => 'sql',
+        'name' => $this->getPatchPath('20131206.phragmentnull.sql'),
+      ),
    );
  }
 }