1
0
Fork 0
mirror of https://we.phorge.it/source/phorge.git synced 2024-11-25 16:22:43 +01:00

Add a basic pull event log for debugging repository cloning

Summary:
Ref T10228. This is currently quite limited:

  - No UI.
  - No SSH support.

My primary goal is to debug the issue in T10228. In the long run we can expand this to be a bit fancier.

Test Plan:
Made various valid and invalid clones, got sucess responses and not-so-successful responses, viewed the log table for general corresponding messages and broad sanity.

Ran GC via `bin/phd debug trigger`, no issues.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T10228

Differential Revision: https://secure.phabricator.com/D15127
This commit is contained in:
epriestley 2016-01-28 05:21:38 -08:00
parent c00cd5c2a3
commit 30473549ac
7 changed files with 371 additions and 0 deletions

View file

@ -0,0 +1,14 @@
CREATE TABLE {$NAMESPACE}_repository.repository_pullevent (
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
phid VARBINARY(64) NOT NULL,
repositoryPHID VARBINARY(64),
epoch INT UNSIGNED NOT NULL,
pullerPHID VARBINARY(64),
remoteAddress INT UNSIGNED,
remoteProtocol VARCHAR(32) COLLATE {$COLLATE_TEXT},
resultType VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT},
resultCode INT UNSIGNED NOT NULL,
properties LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT},
KEY `key_repository` (repositoryPHID),
KEY `key_epoch` (epoch)
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

View file

@ -688,6 +688,7 @@ phutil_register_library_map(array(
'DiffusionPreCommitRefRepositoryHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitRefRepositoryHeraldField.php',
'DiffusionPreCommitRefRepositoryProjectsHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitRefRepositoryProjectsHeraldField.php',
'DiffusionPreCommitRefTypeHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitRefTypeHeraldField.php',
'DiffusionPullEventGarbageCollector' => 'applications/diffusion/DiffusionPullEventGarbageCollector.php',
'DiffusionPushCapability' => 'applications/diffusion/capability/DiffusionPushCapability.php',
'DiffusionPushEventViewController' => 'applications/diffusion/controller/DiffusionPushEventViewController.php',
'DiffusionPushLogController' => 'applications/diffusion/controller/DiffusionPushLogController.php',
@ -3028,6 +3029,9 @@ phutil_register_library_map(array(
'PhabricatorRepositoryMirrorQuery' => 'applications/repository/query/PhabricatorRepositoryMirrorQuery.php',
'PhabricatorRepositoryParsedChange' => 'applications/repository/data/PhabricatorRepositoryParsedChange.php',
'PhabricatorRepositoryPullEngine' => 'applications/repository/engine/PhabricatorRepositoryPullEngine.php',
'PhabricatorRepositoryPullEvent' => 'applications/repository/storage/PhabricatorRepositoryPullEvent.php',
'PhabricatorRepositoryPullEventPHIDType' => 'applications/repository/phid/PhabricatorRepositoryPullEventPHIDType.php',
'PhabricatorRepositoryPullEventQuery' => 'applications/repository/query/PhabricatorRepositoryPullEventQuery.php',
'PhabricatorRepositoryPullLocalDaemon' => 'applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php',
'PhabricatorRepositoryPushEvent' => 'applications/repository/storage/PhabricatorRepositoryPushEvent.php',
'PhabricatorRepositoryPushEventPHIDType' => 'applications/repository/phid/PhabricatorRepositoryPushEventPHIDType.php',
@ -4708,6 +4712,7 @@ phutil_register_library_map(array(
'DiffusionPreCommitRefRepositoryHeraldField' => 'DiffusionPreCommitRefHeraldField',
'DiffusionPreCommitRefRepositoryProjectsHeraldField' => 'DiffusionPreCommitRefHeraldField',
'DiffusionPreCommitRefTypeHeraldField' => 'DiffusionPreCommitRefHeraldField',
'DiffusionPullEventGarbageCollector' => 'PhabricatorGarbageCollector',
'DiffusionPushCapability' => 'PhabricatorPolicyCapability',
'DiffusionPushEventViewController' => 'DiffusionPushLogController',
'DiffusionPushLogController' => 'DiffusionController',
@ -7476,6 +7481,12 @@ phutil_register_library_map(array(
'PhabricatorRepositoryMirrorQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
'PhabricatorRepositoryParsedChange' => 'Phobject',
'PhabricatorRepositoryPullEngine' => 'PhabricatorRepositoryEngine',
'PhabricatorRepositoryPullEvent' => array(
'PhabricatorRepositoryDAO',
'PhabricatorPolicyInterface',
),
'PhabricatorRepositoryPullEventPHIDType' => 'PhabricatorPHIDType',
'PhabricatorRepositoryPullEventQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
'PhabricatorRepositoryPullLocalDaemon' => 'PhabricatorDaemon',
'PhabricatorRepositoryPushEvent' => array(
'PhabricatorRepositoryDAO',

View file

@ -0,0 +1,29 @@
<?php
final class DiffusionPullEventGarbageCollector
extends PhabricatorGarbageCollector {
const COLLECTORCONST = 'diffusion.pull';
public function getCollectorName() {
return pht('Repository Pull Events');
}
public function getDefaultRetentionPolicy() {
return phutil_units('30 days in seconds');
}
protected function collectGarbage() {
$table = new PhabricatorRepositoryPullEvent();
$conn_w = $table->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE epoch < %d LIMIT 100',
$table->getTableName(),
$this->getGarbageEpoch());
return ($conn_w->getAffectedRows() == 100);
}
}

View file

@ -2,6 +2,27 @@
final class DiffusionServeController extends DiffusionController {
private $serviceViewer;
private $serviceRepository;
public function setServiceViewer(PhabricatorUser $viewer) {
$this->serviceViewer = $viewer;
return $this;
}
public function getServiceViewer() {
return $this->serviceViewer;
}
public function setServiceRepository(PhabricatorRepository $repository) {
$this->serviceRepository = $repository;
return $this;
}
public function getServiceRepository() {
return $this->serviceRepository;
}
public function isVCSRequest(AphrontRequest $request) {
$identifier = $this->getRepositoryIdentifierFromRequest($request);
if ($identifier === null) {
@ -45,6 +66,75 @@ final class DiffusionServeController extends DiffusionController {
}
public function handleRequest(AphrontRequest $request) {
$service_exception = null;
$response = null;
try {
$response = $this->serveRequest($request);
} catch (Exception $ex) {
$service_exception = $ex;
}
try {
$remote_addr = $request->getRemoteAddr();
$remote_addr = ip2long($remote_addr);
$pull_event = id(new PhabricatorRepositoryPullEvent())
->setEpoch(PhabricatorTime::getNow())
->setRemoteAddress($remote_addr)
->setRemoteProtocol('http');
if ($response) {
$pull_event
->setResultType('wild')
->setResultCode($response->getHTTPResponseCode());
if ($response instanceof PhabricatorVCSResponse) {
$pull_event->setProperties(
array(
'response.message' => $response->getMessage(),
));
}
} else {
$pull_event
->setResultType('exception')
->setResultCode(500)
->setProperties(
array(
'exception.class' => $ex->getClass(),
'exception.message' => $ex->getMessage(),
));
}
$viewer = $this->getServiceViewer();
if ($viewer) {
$pull_event->setPullerPHID($viewer->getPHID());
}
$repository = $this->getServiceRepository();
if ($repository) {
$pull_event->setRepositoryPHID($repository->getPHID());
}
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
$pull_event->save();
unset($unguarded);
} catch (Exception $ex) {
if ($service_exception) {
throw $service_exception;
}
throw $ex;
}
if ($service_exception) {
throw $service_exception;
}
return $response;
}
private function serveRequest(AphrontRequest $request) {
$identifier = $this->getRepositoryIdentifierFromRequest($request);
// If authentication credentials have been provided, try to find a user
@ -65,6 +155,8 @@ final class DiffusionServeController extends DiffusionController {
$viewer = new PhabricatorUser();
}
$this->setServiceViewer($viewer);
$allow_public = PhabricatorEnv::getEnvConfig('policy.allow-public');
$allow_auth = PhabricatorEnv::getEnvConfig('diffusion.allow-http-auth');
if (!$allow_public) {
@ -111,6 +203,8 @@ final class DiffusionServeController extends DiffusionController {
}
}
$this->setServiceRepository($repository);
if (!$repository->isTracked()) {
return new PhabricatorVCSResponse(
403,

View file

@ -0,0 +1,39 @@
<?php
final class PhabricatorRepositoryPullEventPHIDType extends PhabricatorPHIDType {
const TYPECONST = 'PULE';
public function getTypeName() {
return pht('Pull Event');
}
public function newObject() {
return new PhabricatorRepositoryPullEvent();
}
public function getPHIDTypeApplicationClass() {
return 'PhabricatorDiffusionApplication';
}
protected function buildQueryForObjects(
PhabricatorObjectQuery $query,
array $phids) {
return id(new PhabricatorRepositoryPullEventQuery())
->withPHIDs($phids);
}
public function loadHandles(
PhabricatorHandleQuery $query,
array $handles,
array $objects) {
foreach ($handles as $phid => $handle) {
$event = $objects[$phid];
$handle->setName(pht('Pull Event %d', $event->getID()));
}
}
}

View file

@ -0,0 +1,97 @@
<?php
final class PhabricatorRepositoryPullEventQuery
extends PhabricatorCursorPagedPolicyAwareQuery {
private $ids;
private $phids;
private $repositoryPHIDs;
private $pullerPHIDs;
public function withIDs(array $ids) {
$this->ids = $ids;
return $this;
}
public function withPHIDs(array $phids) {
$this->phids = $phids;
return $this;
}
public function withRepositoryPHIDs(array $repository_phids) {
$this->repositoryPHIDs = $repository_phids;
return $this;
}
public function withPullerPHIDs(array $puller_phids) {
$this->pullerPHIDs = $puller_phids;
return $this;
}
public function newResultObject() {
return new PhabricatorRepositoryPullEvent();
}
protected function loadPage() {
return $this->loadStandardPage($this->newResultObject());
}
protected function willFilterPage(array $events) {
$repository_phids = mpull($events, 'getRepositoryPHID');
$repositories = id(new PhabricatorRepositoryQuery())
->setViewer($this->getViewer())
->withPHIDs($repository_phids)
->execute();
$repositories = mpull($repositories, null, 'getPHID');
foreach ($events as $key => $event) {
$phid = $event->getRepositoryPHID();
if (empty($repositories[$phid])) {
unset($events[$key]);
continue;
}
$event->attachRepository($repositories[$phid]);
}
return $events;
}
protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) {
$where = parent::buildWhereClauseParts($conn);
if ($this->ids !== null) {
$where[] = qsprintf(
$conn,
'id IN (%Ld)',
$this->ids);
}
if ($this->phids !== null) {
$where[] = qsprintf(
$conn,
'phid IN (%Ls)',
$this->phids);
}
if ($this->repositoryPHIDs !== null) {
$where[] = qsprintf(
$conn,
'repositoryPHID IN (%Ls)',
$this->repositoryPHIDs);
}
if ($this->pullerPHIDs !== null) {
$where[] = qsprintf(
$conn,
'pullerPHID in (%Ls)',
$this->pullerPHIDs);
}
return $where;
}
public function getQueryApplicationClass() {
return 'PhabricatorDiffusionApplication';
}
}

View file

@ -0,0 +1,87 @@
<?php
final class PhabricatorRepositoryPullEvent
extends PhabricatorRepositoryDAO
implements PhabricatorPolicyInterface {
protected $repositoryPHID;
protected $epoch;
protected $pullerPHID;
protected $remoteAddress;
protected $remoteProtocol;
protected $resultType;
protected $resultCode;
protected $properties;
private $repository = self::ATTACHABLE;
public static function initializeNewEvent(PhabricatorUser $viewer) {
return id(new PhabricatorRepositoryPushEvent())
->setPusherPHID($viewer->getPHID());
}
protected function getConfiguration() {
return array(
self::CONFIG_AUX_PHID => true,
self::CONFIG_TIMESTAMPS => false,
self::CONFIG_SERIALIZATION => array(
'properties' => self::SERIALIZATION_JSON,
),
self::CONFIG_COLUMN_SCHEMA => array(
'repositoryPHID' => 'phid?',
'pullerPHID' => 'phid?',
'remoteAddress' => 'uint32?',
'remoteProtocol' => 'text32?',
'resultType' => 'text32',
'resultCode' => 'uint32',
),
self::CONFIG_KEY_SCHEMA => array(
'key_repository' => array(
'columns' => array('repositoryPHID'),
),
'key_epoch' => array(
'columns' => array('epoch'),
),
),
) + parent::getConfiguration();
}
public function generatePHID() {
return PhabricatorPHID::generateNewPHID(
PhabricatorRepositoryPullEventPHIDType::TYPECONST);
}
public function attachRepository(PhabricatorRepository $repository) {
$this->repository = $repository;
return $this;
}
public function getRepository() {
return $this->assertAttached($this->repository);
}
/* -( PhabricatorPolicyInterface )----------------------------------------- */
public function getCapabilities() {
return array(
PhabricatorPolicyCapability::CAN_VIEW,
);
}
public function getPolicy($capability) {
return $this->getRepository()->getPolicy($capability);
}
public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
return $this->getRepository()->hasAutomaticCapability($capability, $viewer);
}
public function describeAutomaticCapability($capability) {
return pht(
"A repository's pull events are visible to users who can see the ".
"repository.");
}
}