Add a basic push log for recording repository push events

Summary:
Ref T4195. This log serves two purposes:

  - It's a log, so you can see what happened. Particularly, in Git/Hg, there is no other way to tell:
    - Who //pushed// a change (vs committed / authored)?
    - When was a change pushed?
    - What was the old value of some tag/branch before someone destroyed it?
  - We can hand these objects off to Herald to implement pre-commit rules.

This is a very basic implementation, but gets some data written and has a basic UI for it.

Test Plan: {F87339}

Reviewers: btrahan

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T4195

Differential Revision: https://secure.phabricator.com/D7705

resources/sql/patches/20131204.pushlog.sql

@@ -0,0 +1,25 @@
+CREATE TABLE {$NAMESPACE}_repository.repository_pushlog (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  epoch INT UNSIGNED NOT NULL,
+  repositoryPHID VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  pusherPHID VARCHAR(64) NOT NULL COLLATE utf8_bin,
+  remoteAddress INT UNSIGNED,
+  remoteProtocol VARCHAR(32),
+  transactionKey CHAR(12) NOT NULL COLLATE latin1_bin,
+  refType VARCHAR(12) NOT NULL COLLATE utf8_bin,
+  refNameHash VARCHAR(12) COLLATE latin1_bin,
+  refNameRaw LONGTEXT COLLATE latin1_bin,
+  refNameEncoding VARCHAR(16) COLLATE utf8_bin,
+  refOld VARCHAR(40) COLLATE latin1_bin,
+  refNew VARCHAR(40) NOT NULL COLLATE latin1_bin,
+  mergeBase VARCHAR(40) COLLATE latin1_bin,
+  changeFlags INT UNSIGNED NOT NULL,
+  rejectCode INT UNSIGNED NOT NULL,
+  rejectDetails VARCHAR(64) COLLATE utf8_bin,
+
+  KEY `key_repository` (repositoryPHID),
+  KEY `key_ref` (repositoryPHID, refNew),
+  KEY `key_pusher` (pusherPHID),
+  KEY `key_name` (repositoryPHID, refNameHash)
+
+) ENGINE=InnoDB, COLLATE=utf8_general_ci;

src/__phutil_library_map__.php

@@ -522,6 +522,7 @@ phutil_register_library_map(array(
     'DiffusionPathQuery' => 'applications/diffusion/query/DiffusionPathQuery.php',
     'DiffusionPathQueryTestCase' => 'applications/diffusion/query/pathid/__tests__/DiffusionPathQueryTestCase.php',
     'DiffusionPathValidateController' => 'applications/diffusion/controller/DiffusionPathValidateController.php',
+    'DiffusionPushLogListController' => 'applications/diffusion/controller/DiffusionPushLogListController.php',
     'DiffusionQuery' => 'applications/diffusion/query/DiffusionQuery.php',
     'DiffusionRawDiffQuery' => 'applications/diffusion/query/rawdiff/DiffusionRawDiffQuery.php',
     'DiffusionRemarkupRule' => 'applications/diffusion/remarkup/DiffusionRemarkupRule.php',
@@ -1793,6 +1794,9 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryPHIDTypeRepository' => 'applications/repository/phid/PhabricatorRepositoryPHIDTypeRepository.php',
     'PhabricatorRepositoryPullEngine' => 'applications/repository/engine/PhabricatorRepositoryPullEngine.php',
     'PhabricatorRepositoryPullLocalDaemon' => 'applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php',
+    'PhabricatorRepositoryPushLog' => 'applications/repository/storage/PhabricatorRepositoryPushLog.php',
+    'PhabricatorRepositoryPushLogQuery' => 'applications/repository/query/PhabricatorRepositoryPushLogQuery.php',
+    'PhabricatorRepositoryPushLogSearchEngine' => 'applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php',
     'PhabricatorRepositoryQuery' => 'applications/repository/query/PhabricatorRepositoryQuery.php',
     'PhabricatorRepositorySearchEngine' => 'applications/repository/query/PhabricatorRepositorySearchEngine.php',
     'PhabricatorRepositoryShortcut' => 'applications/repository/storage/PhabricatorRepositoryShortcut.php',
@@ -2848,6 +2852,11 @@ phutil_register_library_map(array(
     'DiffusionPathCompleteController' => 'DiffusionController',
     'DiffusionPathQueryTestCase' => 'PhabricatorTestCase',
     'DiffusionPathValidateController' => 'DiffusionController',
+    'DiffusionPushLogListController' =>
+    array(
+      0 => 'DiffusionController',
+      1 => 'PhabricatorApplicationSearchResultsControllerInterface',
+    ),
     'DiffusionQuery' => 'PhabricatorQuery',
     'DiffusionRawDiffQuery' => 'DiffusionQuery',
     'DiffusionRemarkupRule' => 'PhabricatorRemarkupRuleObject',
@@ -4309,6 +4318,13 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryPHIDTypeRepository' => 'PhabricatorPHIDType',
     'PhabricatorRepositoryPullEngine' => 'PhabricatorRepositoryEngine',
     'PhabricatorRepositoryPullLocalDaemon' => 'PhabricatorDaemon',
+    'PhabricatorRepositoryPushLog' =>
+    array(
+      0 => 'PhabricatorRepositoryDAO',
+      1 => 'PhabricatorPolicyInterface',
+    ),
+    'PhabricatorRepositoryPushLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
+    'PhabricatorRepositoryPushLogSearchEngine' => 'PhabricatorApplicationSearchEngine',
     'PhabricatorRepositoryQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'PhabricatorRepositorySearchEngine' => 'PhabricatorApplicationSearchEngine',
     'PhabricatorRepositoryShortcut' => 'PhabricatorRepositoryDAO',

src/applications/diffusion/application/PhabricatorApplicationDiffusion.php

@@ -46,6 +46,9 @@ final class PhabricatorApplicationDiffusion extends PhabricatorApplication {
         'new/' => 'DiffusionRepositoryNewController',
         '(?P<edit>create)/' => 'DiffusionRepositoryCreateController',
         '(?P<edit>import)/' => 'DiffusionRepositoryCreateController',
+        'pushlog/(?:query/(?P<queryKey>[^/]+)/)?'
+          => 'DiffusionPushLogListController',
+
         '(?P<callsign>[A-Z]+)/' => array(
           '' => 'DiffusionRepositoryController',
 
@@ -58,7 +61,6 @@ final class PhabricatorApplicationDiffusion extends PhabricatorApplication {
           'tags/(?P<dblob>.*)'          => 'DiffusionTagListController',
           'branches/(?P<dblob>.*)'      => 'DiffusionBranchTableController',
           'lint/(?P<dblob>.*)'          => 'DiffusionLintController',
-
           'commit/(?P<commit>[a-z0-9]+)/branches/'
             => 'DiffusionCommitBranchesController',
           'commit/(?P<commit>[a-z0-9]+)/tags/'

src/applications/diffusion/controller/DiffusionPushLogListController.php

@@ -0,0 +1,96 @@
+<?php
+
+final class DiffusionPushLogListController extends DiffusionController
+  implements PhabricatorApplicationSearchResultsControllerInterface {
+
+  private $queryKey;
+
+  public function shouldAllowPublic() {
+    return true;
+  }
+
+  public function willProcessRequest(array $data) {
+    $this->queryKey = idx($data, 'queryKey');
+  }
+
+  public function processRequest() {
+    $request = $this->getRequest();
+    $controller = id(new PhabricatorApplicationSearchController($request))
+      ->setQueryKey($this->queryKey)
+      ->setSearchEngine(new PhabricatorRepositoryPushLogSearchEngine())
+      ->setNavigation($this->buildSideNavView());
+
+    return $this->delegateToController($controller);
+  }
+
+  public function renderResultsList(
+    array $logs,
+    PhabricatorSavedQuery $query) {
+    $viewer = $this->getRequest()->getUser();
+
+    $this->loadHandles(mpull($logs, 'getPusherPHID'));
+
+    $rows = array();
+    foreach ($logs as $log) {
+      $callsign = $log->getRepository()->getCallsign();
+      $rows[] = array(
+        phutil_tag(
+          'a',
+          array(
+            'href' => $this->getApplicationURI($callsign.'/'),
+          ),
+          $callsign),
+        $this->getHandle($log->getPusherPHID())->renderLink(),
+        $log->getRefType(),
+        $log->getRefName(),
+        $log->getRefOldShort(),
+        $log->getRefNewShort(),
+        phabricator_datetime($log->getEpoch(), $viewer),
+      );
+    }
+
+    $table = id(new AphrontTableView($rows))
+      ->setHeaders(
+        array(
+          pht('Repository'),
+          pht('Pusher'),
+          pht('Type'),
+          pht('Name'),
+          pht('Old'),
+          pht('New'),
+          pht('Date'),
+        ))
+      ->setColumnClasses(
+        array(
+          '',
+          '',
+          '',
+          'wide',
+          'n',
+          'n',
+          'date',
+        ));
+
+    $box = id(new PHUIBoxView())
+      ->addMargin(PHUI::MARGIN_LARGE)
+      ->appendChild($table);
+
+    return $box;
+  }
+
+  public function buildSideNavView($for_app = false) {
+    $viewer = $this->getRequest()->getUser();
+
+    $nav = new AphrontSideNavFilterView();
+    $nav->setBaseURI(new PhutilURI($this->getApplicationURI()));
+
+    id(new PhabricatorRepositoryPushLogSearchEngine())
+      ->setViewer($viewer)
+      ->addNavigationItems($nav->getMenu());
+
+    $nav->selectFilter(null);
+
+    return $nav;
+  }
+
+}

src/applications/diffusion/engine/DiffusionCommitHookEngine.php

@@ -1,5 +1,10 @@
 <?php
 
+/**
+ * @task git  Git Hooks
+ * @task hg   Mercurial Hooks
+ * @task svn  Subversion Hooks
+ */
 final class DiffusionCommitHookEngine extends Phobject {
 
   private $viewer;
@@ -77,6 +82,48 @@ final class DiffusionCommitHookEngine extends Phobject {
 
     // TODO: Now, do content checks.
 
+    // TODO: Generalize this; just getting some data in the database for now.
+    $transaction_key = PhabricatorHash::digestForIndex(
+      Filesystem::readRandomBytes(64));
+
+    $logs = array();
+    foreach ($updates as $update) {
+      $log = PhabricatorRepositoryPushLog::initializeNewLog($this->getViewer())
+        ->setRepositoryPHID($this->getRepository()->getPHID())
+        ->setEpoch(time())
+        ->setRemoteAddress(null) // TODO: Populate this where possible.
+        ->setRemoteProtocol(null) // TODO: Populate this where possible.
+        ->setTransactionKey($transaction_key)
+        ->setRefType($update['type'])
+        ->setRefNameHash(PhabricatorHash::digestForIndex($update['ref']))
+        ->setRefNameRaw($update['ref'])
+        ->setRefNameEncoding(phutil_is_utf8($update['ref']) ? 'utf8' : null)
+        ->setRefOld($update['old'])
+        ->setRefNew($update['new'])
+        ->setMergeBase(idx($update, 'merge-base'))
+        ->setRejectCode(PhabricatorRepositoryPushLog::REJECT_ACCEPT)
+        ->setRejectDetails(null);
+
+      $flags = 0;
+      if ($update['operation'] == 'create') {
+        $flags = $flags | PhabricatorRepositoryPushLog::CHANGEFLAG_ADD;
+      } else if ($update['operation'] == 'delete') {
+        $flags = $flags | PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE;
+      } else {
+        // TODO: This isn't correct; these might be APPEND or REWRITE, and
+        // if they're REWRITE they might be DANGEROUS. Fix this when this
+        // gets generalized.
+        $flags = $flags | PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND;
+      }
+
+      $log->setChangeFlags($flags);
+      $logs[] = $log;
+    }
+
+    foreach ($logs as $log) {
+      $log->save();
+    }
+
     return 0;
   }
 

src/applications/repository/query/PhabricatorRepositoryPushLogQuery.php

@@ -0,0 +1,86 @@
+<?php
+
+final class PhabricatorRepositoryPushLogQuery
+  extends PhabricatorCursorPagedPolicyAwareQuery {
+
+  private $ids;
+  private $repositoryPHIDs;
+
+  public function withIDs(array $ids) {
+    $this->ids = $ids;
+    return $this;
+  }
+
+  public function withRepositoryPHIDs(array $repository_phids) {
+    $this->repositoryPHIDs = $repository_phids;
+    return $this;
+  }
+
+  protected function loadPage() {
+    $table = new PhabricatorRepositoryPushLog();
+    $conn_r = $table->establishConnection('r');
+
+    $data = queryfx_all(
+      $conn_r,
+      'SELECT * FROM %T %Q %Q %Q',
+      $table->getTableName(),
+      $this->buildWhereClause($conn_r),
+      $this->buildOrderClause($conn_r),
+      $this->buildLimitClause($conn_r));
+
+    return $table->loadAllFromArray($data);
+  }
+
+  public function willFilterPage(array $logs) {
+    $repository_phids = mpull($logs, 'getRepositoryPHID');
+    if ($repository_phids) {
+      $repositories = id(new PhabricatorRepositoryQuery())
+        ->setViewer($this->getViewer())
+        ->withPHIDs($repository_phids)
+        ->execute();
+      $repositories = mpull($repositories, null, 'getPHID');
+    } else {
+      $repositories = array();
+    }
+
+    foreach ($logs as $key => $log) {
+      $phid = $log->getRepositoryPHID();
+      if (empty($repositories[$phid])) {
+        unset($logs[$key]);
+        continue;
+      }
+      $log->attachRepository($repositories[$phid]);
+    }
+
+    return $logs;
+  }
+
+
+  private function buildWhereClause(AphrontDatabaseConnection $conn_r) {
+    $where = array();
+
+    if ($this->ids) {
+      $where[] = qsprintf(
+        $conn_r,
+        'id IN (%Ld)',
+        $this->ids);
+    }
+
+    if ($this->repositoryPHIDs) {
+      $where[] = qsprintf(
+        $conn_r,
+        'repositoryPHID IN (%Ls)',
+        $this->repositoryPHIDs);
+    }
+
+    $where[] = $this->buildPagingClause($conn_r);
+
+    return $this->formatWhereClause($where);
+  }
+
+
+  public function getQueryApplicationClass() {
+    return 'PhabricatorApplicationDiffusion';
+  }
+
+}

src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php

@@ -0,0 +1,49 @@
+<?php
+
+final class PhabricatorRepositoryPushLogSearchEngine
+  extends PhabricatorApplicationSearchEngine {
+
+  public function buildSavedQueryFromRequest(AphrontRequest $request) {
+    $saved = new PhabricatorSavedQuery();
+
+    return $saved;
+  }
+
+  public function buildQueryFromSavedQuery(PhabricatorSavedQuery $saved) {
+    $query = id(new PhabricatorRepositoryPushLogQuery());
+
+    return $query;
+  }
+
+  public function buildSearchForm(
+    AphrontFormView $form,
+    PhabricatorSavedQuery $saved_query) {
+
+  }
+
+  protected function getURI($path) {
+    return '/diffusion/pushlog/'.$path;
+  }
+
+  public function getBuiltinQueryNames() {
+    $names = array(
+      'all' => pht('All Push Logs'),
+    );
+
+    return $names;
+  }
+
+  public function buildSavedQueryFromBuiltin($query_key) {
+
+    $query = $this->newSavedQuery();
+    $query->setQueryKey($query_key);
+
+    switch ($query_key) {
+      case 'all':
+        return $query;
+    }
+
+    return parent::buildSavedQueryFromBuiltin($query_key);
+  }
+
+}

src/applications/repository/storage/PhabricatorRepositoryPushLog.php

@@ -0,0 +1,117 @@
+<?php
+
+/**
+ * Records a push to a hosted repository. This allows us to store metadata
+ * about who pushed commits, when, and from where. We can also record the
+ * history of branches and tags, which is not normally persisted outside of
+ * the reflog.
+ *
+ * This log is written by commit hooks installed into hosted repositories.
+ * See @{class:DiffusionCommitHookEngine}.
+ */
+final class PhabricatorRepositoryPushLog
+  extends PhabricatorRepositoryDAO
+  implements PhabricatorPolicyInterface {
+
+  const REFTYPE_BRANCH = 'branch';
+  const REFTYPE_TAG = 'tag';
+  const REFTYPE_BOOKMARK = 'bookmark';
+  const REFTYPE_SVN = 'svn';
+  const REFTYPE_COMMIT = 'commit';
+
+  const CHANGEFLAG_ADD = 1;
+  const CHANGEFLAG_DELETE = 2;
+  const CHANGEFLAG_APPEND = 4;
+  const CHANGEFLAG_REWRITE = 8;
+  const CHANGEFLAG_DANGEROUS = 16;
+
+  const REJECT_ACCEPT = 0;
+  const REJECT_DANGEROUS = 1;
+  const REJECT_HERALD = 2;
+  const REJECT_EXTERNAL = 3;
+
+  protected $repositoryPHID;
+  protected $epoch;
+  protected $pusherPHID;
+  protected $remoteAddress;
+  protected $remoteProtocol;
+  protected $transactionKey;
+  protected $refType;
+  protected $refNameHash;
+  protected $refNameRaw;
+  protected $refNameEncoding;
+  protected $refOld;
+  protected $refNew;
+  protected $mergeBase;
+  protected $changeFlags;
+  protected $rejectCode;
+  protected $rejectDetails;
+
+  private $repository = self::ATTACHABLE;
+
+  public static function initializeNewLog(PhabricatorUser $viewer) {
+    return id(new PhabricatorRepositoryPushLog())
+      ->setPusherPHID($viewer->getPHID());
+  }
+
+  public function getConfiguration() {
+    return array(
+      self::CONFIG_TIMESTAMPS => false,
+    ) + parent::getConfiguration();
+  }
+
+  public function attachRepository(PhabricatorRepository $repository) {
+    $this->repository = $repository;
+    return $this;
+  }
+
+  public function getRepository() {
+    return $this->assertAttached($this->repository);
+  }
+
+  public function getRefName() {
+    if ($this->getRefNameEncoding() == 'utf8') {
+      return $this->getRefNameRaw();
+    }
+    return phutil_utf8ize($this->getRefNameRaw());
+  }
+
+  public function getRefOldShort() {
+    if ($this->getRepository()->isSVN()) {
+      return $this->getRefOld();
+    }
+    return substr($this->getRefOld(), 0, 12);
+  }
+
+  public function getRefNewShort() {
+    if ($this->getRepository()->isSVN()) {
+      return $this->getRefNew();
+    }
+    return substr($this->getRefNew(), 0, 12);
+  }
+
+
+/* -(  PhabricatorPolicyInterface  )----------------------------------------- */
+
+
+  public function getCapabilities() {
+    return array(
+      PhabricatorPolicyCapability::CAN_VIEW,
+    );
+  }
+
+  public function getPolicy($capability) {
+    return $this->getRepository()->getPolicy($capability);
+  }
+
+  public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
+    return $this->getRepository()->hasAutomaticCapability($capability, $viewer);
+  }
+
+  public function describeAutomaticCapability($capability) {
+    return pht(
+      "A repository's push logs are visible to users who can see the ".
+      "repository.");
+  }
+
+}

src/docs/user/userguide/diffusion_hosting.diviner

@@ -31,8 +31,10 @@ performant, but HTTP is easier to set up and supports anonymous access.
 | Reads | Yes | Yes |
 | Writes | Yes | Yes |
 | Authenticated Access | Yes | Yes |
+| Push Logs | Yes | Yes |
+| Commit Hooks | Yes | Yes |
 | Anonymous Access | No | Yes |
-| Security | Better (Asymetric Key) | Okay (Password) |
+| Security | Better (Asymmetric Key) | Okay (Password) |
 | Performance | Better | Okay |
 | Setup | Hard | Easy |
 

src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php

@@ -1800,6 +1800,10 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList {
         'type' => 'sql',
         'name' => $this->getPatchPath('20131205.buildtargets.sql'),
       ),
+      '20131204.pushlog.sql' => array(
+        'type' => 'sql',
+        'name' => $this->getPatchPath('20131204.pushlog.sql'),
+      ),
     );
   }
 }