Add a basic pull event log for debugging repository cloning

Summary:
Ref T10228. This is currently quite limited:

  - No UI.
  - No SSH support.

My primary goal is to debug the issue in T10228. In the long run we can expand this to be a bit fancier.

Test Plan:
Made various valid and invalid clones, got sucess responses and not-so-successful responses, viewed the log table for general corresponding messages and broad sanity.

Ran GC via `bin/phd debug trigger`, no issues.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T10228

Differential Revision: https://secure.phabricator.com/D15127

resources/sql/autopatches/20160128.repo.1.pull.sql

@@ -0,0 +1,14 @@
+CREATE TABLE {$NAMESPACE}_repository.repository_pullevent (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+  phid VARBINARY(64) NOT NULL,
+  repositoryPHID VARBINARY(64),
+  epoch INT UNSIGNED NOT NULL,
+  pullerPHID VARBINARY(64),
+  remoteAddress INT UNSIGNED,
+  remoteProtocol VARCHAR(32) COLLATE {$COLLATE_TEXT},
+  resultType VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT},
+  resultCode INT UNSIGNED NOT NULL,
+  properties LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT},
+  KEY `key_repository` (repositoryPHID),
+  KEY `key_epoch` (epoch)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

src/__phutil_library_map__.php

@@ -688,6 +688,7 @@ phutil_register_library_map(array(
     'DiffusionPreCommitRefRepositoryHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitRefRepositoryHeraldField.php',
     'DiffusionPreCommitRefRepositoryProjectsHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitRefRepositoryProjectsHeraldField.php',
     'DiffusionPreCommitRefTypeHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitRefTypeHeraldField.php',
+    'DiffusionPullEventGarbageCollector' => 'applications/diffusion/DiffusionPullEventGarbageCollector.php',
     'DiffusionPushCapability' => 'applications/diffusion/capability/DiffusionPushCapability.php',
     'DiffusionPushEventViewController' => 'applications/diffusion/controller/DiffusionPushEventViewController.php',
     'DiffusionPushLogController' => 'applications/diffusion/controller/DiffusionPushLogController.php',
@@ -3028,6 +3029,9 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryMirrorQuery' => 'applications/repository/query/PhabricatorRepositoryMirrorQuery.php',
     'PhabricatorRepositoryParsedChange' => 'applications/repository/data/PhabricatorRepositoryParsedChange.php',
     'PhabricatorRepositoryPullEngine' => 'applications/repository/engine/PhabricatorRepositoryPullEngine.php',
+    'PhabricatorRepositoryPullEvent' => 'applications/repository/storage/PhabricatorRepositoryPullEvent.php',
+    'PhabricatorRepositoryPullEventPHIDType' => 'applications/repository/phid/PhabricatorRepositoryPullEventPHIDType.php',
+    'PhabricatorRepositoryPullEventQuery' => 'applications/repository/query/PhabricatorRepositoryPullEventQuery.php',
     'PhabricatorRepositoryPullLocalDaemon' => 'applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php',
     'PhabricatorRepositoryPushEvent' => 'applications/repository/storage/PhabricatorRepositoryPushEvent.php',
     'PhabricatorRepositoryPushEventPHIDType' => 'applications/repository/phid/PhabricatorRepositoryPushEventPHIDType.php',
@@ -4708,6 +4712,7 @@ phutil_register_library_map(array(
     'DiffusionPreCommitRefRepositoryHeraldField' => 'DiffusionPreCommitRefHeraldField',
     'DiffusionPreCommitRefRepositoryProjectsHeraldField' => 'DiffusionPreCommitRefHeraldField',
     'DiffusionPreCommitRefTypeHeraldField' => 'DiffusionPreCommitRefHeraldField',
+    'DiffusionPullEventGarbageCollector' => 'PhabricatorGarbageCollector',
     'DiffusionPushCapability' => 'PhabricatorPolicyCapability',
     'DiffusionPushEventViewController' => 'DiffusionPushLogController',
     'DiffusionPushLogController' => 'DiffusionController',
@@ -7476,6 +7481,12 @@ phutil_register_library_map(array(
     'PhabricatorRepositoryMirrorQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'PhabricatorRepositoryParsedChange' => 'Phobject',
     'PhabricatorRepositoryPullEngine' => 'PhabricatorRepositoryEngine',
+    'PhabricatorRepositoryPullEvent' => array(
+      'PhabricatorRepositoryDAO',
+      'PhabricatorPolicyInterface',
+    ),
+    'PhabricatorRepositoryPullEventPHIDType' => 'PhabricatorPHIDType',
+    'PhabricatorRepositoryPullEventQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'PhabricatorRepositoryPullLocalDaemon' => 'PhabricatorDaemon',
     'PhabricatorRepositoryPushEvent' => array(
       'PhabricatorRepositoryDAO',

src/applications/diffusion/DiffusionPullEventGarbageCollector.php

@@ -0,0 +1,29 @@
+<?php
+
+final class DiffusionPullEventGarbageCollector
+  extends PhabricatorGarbageCollector {
+
+  const COLLECTORCONST = 'diffusion.pull';
+
+  public function getCollectorName() {
+    return pht('Repository Pull Events');
+  }
+
+  public function getDefaultRetentionPolicy() {
+    return phutil_units('30 days in seconds');
+  }
+
+  protected function collectGarbage() {
+    $table = new PhabricatorRepositoryPullEvent();
+    $conn_w = $table->establishConnection('w');
+
+    queryfx(
+      $conn_w,
+      'DELETE FROM %T WHERE epoch < %d LIMIT 100',
+      $table->getTableName(),
+      $this->getGarbageEpoch());
+
+    return ($conn_w->getAffectedRows() == 100);
+  }
+
+}

src/applications/diffusion/controller/DiffusionServeController.php

@@ -2,6 +2,27 @@
 
 final class DiffusionServeController extends DiffusionController {
 
+  private $serviceViewer;
+  private $serviceRepository;
+
+  public function setServiceViewer(PhabricatorUser $viewer) {
+    $this->serviceViewer = $viewer;
+    return $this;
+  }
+
+  public function getServiceViewer() {
+    return $this->serviceViewer;
+  }
+
+  public function setServiceRepository(PhabricatorRepository $repository) {
+    $this->serviceRepository = $repository;
+    return $this;
+  }
+
+  public function getServiceRepository() {
+    return $this->serviceRepository;
+  }
+
   public function isVCSRequest(AphrontRequest $request) {
     $identifier = $this->getRepositoryIdentifierFromRequest($request);
     if ($identifier === null) {
@@ -45,6 +66,75 @@ final class DiffusionServeController extends DiffusionController {
   }
 
   public function handleRequest(AphrontRequest $request) {
+    $service_exception = null;
+    $response = null;
+
+    try {
+      $response = $this->serveRequest($request);
+    } catch (Exception $ex) {
+      $service_exception = $ex;
+    }
+
+    try {
+      $remote_addr = $request->getRemoteAddr();
+      $remote_addr = ip2long($remote_addr);
+
+      $pull_event = id(new PhabricatorRepositoryPullEvent())
+        ->setEpoch(PhabricatorTime::getNow())
+        ->setRemoteAddress($remote_addr)
+        ->setRemoteProtocol('http');
+
+      if ($response) {
+        $pull_event
+          ->setResultType('wild')
+          ->setResultCode($response->getHTTPResponseCode());
+
+        if ($response instanceof PhabricatorVCSResponse) {
+          $pull_event->setProperties(
+            array(
+              'response.message' => $response->getMessage(),
+            ));
+        }
+      } else {
+        $pull_event
+          ->setResultType('exception')
+          ->setResultCode(500)
+          ->setProperties(
+            array(
+              'exception.class' => $ex->getClass(),
+              'exception.message' => $ex->getMessage(),
+            ));
+      }
+
+      $viewer = $this->getServiceViewer();
+      if ($viewer) {
+        $pull_event->setPullerPHID($viewer->getPHID());
+      }
+
+      $repository = $this->getServiceRepository();
+      if ($repository) {
+        $pull_event->setRepositoryPHID($repository->getPHID());
+      }
+
+      $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
+        $pull_event->save();
+      unset($unguarded);
+
+    } catch (Exception $ex) {
+      if ($service_exception) {
+        throw $service_exception;
+      }
+      throw $ex;
+    }
+
+    if ($service_exception) {
+      throw $service_exception;
+    }
+
+    return $response;
+  }
+
+  private function serveRequest(AphrontRequest $request) {
     $identifier = $this->getRepositoryIdentifierFromRequest($request);
 
     // If authentication credentials have been provided, try to find a user
@@ -65,6 +155,8 @@ final class DiffusionServeController extends DiffusionController {
       $viewer = new PhabricatorUser();
     }
 
+    $this->setServiceViewer($viewer);
+
     $allow_public = PhabricatorEnv::getEnvConfig('policy.allow-public');
     $allow_auth = PhabricatorEnv::getEnvConfig('diffusion.allow-http-auth');
     if (!$allow_public) {
@@ -111,6 +203,8 @@ final class DiffusionServeController extends DiffusionController {
       }
     }
 
+    $this->setServiceRepository($repository);
+
     if (!$repository->isTracked()) {
       return new PhabricatorVCSResponse(
         403,

src/applications/repository/phid/PhabricatorRepositoryPullEventPHIDType.php

@@ -0,0 +1,39 @@
+<?php
+
+final class PhabricatorRepositoryPullEventPHIDType extends PhabricatorPHIDType {
+
+  const TYPECONST = 'PULE';
+
+  public function getTypeName() {
+    return pht('Pull Event');
+  }
+
+  public function newObject() {
+    return new PhabricatorRepositoryPullEvent();
+  }
+
+  public function getPHIDTypeApplicationClass() {
+    return 'PhabricatorDiffusionApplication';
+  }
+
+  protected function buildQueryForObjects(
+    PhabricatorObjectQuery $query,
+    array $phids) {
+
+    return id(new PhabricatorRepositoryPullEventQuery())
+      ->withPHIDs($phids);
+  }
+
+  public function loadHandles(
+    PhabricatorHandleQuery $query,
+    array $handles,
+    array $objects) {
+
+    foreach ($handles as $phid => $handle) {
+      $event = $objects[$phid];
+
+      $handle->setName(pht('Pull Event %d', $event->getID()));
+    }
+  }
+
+}

src/applications/repository/query/PhabricatorRepositoryPullEventQuery.php

@@ -0,0 +1,97 @@
+<?php
+
+final class PhabricatorRepositoryPullEventQuery
+  extends PhabricatorCursorPagedPolicyAwareQuery {
+
+  private $ids;
+  private $phids;
+  private $repositoryPHIDs;
+  private $pullerPHIDs;
+
+  public function withIDs(array $ids) {
+    $this->ids = $ids;
+    return $this;
+  }
+
+  public function withPHIDs(array $phids) {
+    $this->phids = $phids;
+    return $this;
+  }
+
+  public function withRepositoryPHIDs(array $repository_phids) {
+    $this->repositoryPHIDs = $repository_phids;
+    return $this;
+  }
+
+  public function withPullerPHIDs(array $puller_phids) {
+    $this->pullerPHIDs = $puller_phids;
+    return $this;
+  }
+
+  public function newResultObject() {
+    return new PhabricatorRepositoryPullEvent();
+  }
+
+  protected function loadPage() {
+    return $this->loadStandardPage($this->newResultObject());
+  }
+
+  protected function willFilterPage(array $events) {
+    $repository_phids = mpull($events, 'getRepositoryPHID');
+    $repositories = id(new PhabricatorRepositoryQuery())
+      ->setViewer($this->getViewer())
+      ->withPHIDs($repository_phids)
+      ->execute();
+    $repositories = mpull($repositories, null, 'getPHID');
+
+    foreach ($events as $key => $event) {
+      $phid = $event->getRepositoryPHID();
+      if (empty($repositories[$phid])) {
+        unset($events[$key]);
+        continue;
+      }
+      $event->attachRepository($repositories[$phid]);
+    }
+
+    return $events;
+  }
+
+  protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) {
+    $where = parent::buildWhereClauseParts($conn);
+
+    if ($this->ids !== null) {
+      $where[] = qsprintf(
+        $conn,
+        'id IN (%Ld)',
+        $this->ids);
+    }
+
+    if ($this->phids !== null) {
+      $where[] = qsprintf(
+        $conn,
+        'phid IN (%Ls)',
+        $this->phids);
+    }
+
+    if ($this->repositoryPHIDs !== null) {
+      $where[] = qsprintf(
+        $conn,
+        'repositoryPHID IN (%Ls)',
+        $this->repositoryPHIDs);
+    }
+
+    if ($this->pullerPHIDs !== null) {
+      $where[] = qsprintf(
+        $conn,
+        'pullerPHID in (%Ls)',
+        $this->pullerPHIDs);
+    }
+
+    return $where;
+  }
+
+  public function getQueryApplicationClass() {
+    return 'PhabricatorDiffusionApplication';
+  }
+
+}

src/applications/repository/storage/PhabricatorRepositoryPullEvent.php

@@ -0,0 +1,87 @@
+<?php
+
+final class PhabricatorRepositoryPullEvent
+  extends PhabricatorRepositoryDAO
+  implements PhabricatorPolicyInterface {
+
+  protected $repositoryPHID;
+  protected $epoch;
+  protected $pullerPHID;
+  protected $remoteAddress;
+  protected $remoteProtocol;
+  protected $resultType;
+  protected $resultCode;
+  protected $properties;
+
+  private $repository = self::ATTACHABLE;
+
+  public static function initializeNewEvent(PhabricatorUser $viewer) {
+    return id(new PhabricatorRepositoryPushEvent())
+      ->setPusherPHID($viewer->getPHID());
+  }
+
+  protected function getConfiguration() {
+    return array(
+      self::CONFIG_AUX_PHID => true,
+      self::CONFIG_TIMESTAMPS => false,
+      self::CONFIG_SERIALIZATION => array(
+        'properties' => self::SERIALIZATION_JSON,
+      ),
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'repositoryPHID' => 'phid?',
+        'pullerPHID' => 'phid?',
+        'remoteAddress' => 'uint32?',
+        'remoteProtocol' => 'text32?',
+        'resultType' => 'text32',
+        'resultCode' => 'uint32',
+      ),
+      self::CONFIG_KEY_SCHEMA => array(
+        'key_repository' => array(
+          'columns' => array('repositoryPHID'),
+        ),
+        'key_epoch' => array(
+          'columns' => array('epoch'),
+        ),
+      ),
+    ) + parent::getConfiguration();
+  }
+
+  public function generatePHID() {
+    return PhabricatorPHID::generateNewPHID(
+      PhabricatorRepositoryPullEventPHIDType::TYPECONST);
+  }
+
+  public function attachRepository(PhabricatorRepository $repository) {
+    $this->repository = $repository;
+    return $this;
+  }
+
+  public function getRepository() {
+    return $this->assertAttached($this->repository);
+  }
+
+
+/* -(  PhabricatorPolicyInterface  )----------------------------------------- */
+
+
+  public function getCapabilities() {
+    return array(
+      PhabricatorPolicyCapability::CAN_VIEW,
+    );
+  }
+
+  public function getPolicy($capability) {
+    return $this->getRepository()->getPolicy($capability);
+  }
+
+  public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
+    return $this->getRepository()->hasAutomaticCapability($capability, $viewer);
+  }
+
+  public function describeAutomaticCapability($capability) {
+    return pht(
+      "A repository's pull events are visible to users who can see the ".
+      "repository.");
+  }
+
+}