diff --git a/resources/sql/patches/007.daemonlog.sql b/resources/sql/patches/007.daemonlog.sql new file mode 100644 index 0000000000..e1526f1af0 --- /dev/null +++ b/resources/sql/patches/007.daemonlog.sql @@ -0,0 +1,20 @@ +create database phabricator_daemon; + +create table phabricator_daemon.daemon_log ( + id int unsigned not null auto_increment primary key, + daemon varchar(255) not null, + host varchar(255) not null, + pid int unsigned not null, + argv varchar(512) not null, + dateCreated int unsigned not null, + dateModified int unsigned not null +); + +create table phabricator_daemon.daemon_logevent ( + id int unsigned not null auto_increment primary key, + logID int unsigned not null, + logType varchar(4) not null, + message longblob not null, + epoch int unsigned not null, + key (logID, epoch) +); diff --git a/scripts/daemon/phabricator_daemon_launcher.php b/scripts/daemon/phabricator_daemon_launcher.php index fbd36ab2bb..e6c21b255a 100755 --- a/scripts/daemon/phabricator_daemon_launcher.php +++ b/scripts/daemon/phabricator_daemon_launcher.php @@ -40,15 +40,80 @@ switch (isset($argv[1]) ? $argv[1] : 'help') { $err = $control->executeStopCommand(); exit($err); - case 'launch': - phutil_require_module('phutil', 'moduleutils'); + case 'repository-launch-readonly': + $need_launch = phd_load_tracked_repositories_of_type('git'); + if (!$need_launch) { + echo "There are no repositories with tracking enabled.\n"; + } else { + foreach ($need_launch as $repository) { + $name = $repository->getName(); + $callsign = $repository->getCallsign(); + $desc = "'{$name}' ({$callsign})"; + $phid = $repository->getPHID(); + echo "Launching 'git pull' daemon on the {$desc} repository...\n"; + $control->launchDaemon( + 'PhabricatorRepositoryGitPullDaemon', + array( + $phid, + )); + } + } + break; + + case 'repository-launch-master': + $need_launch = phd_load_tracked_repositories(); + if (!$need_launch) { + echo "There are no repositories with tracking enabled.\n"; + } else { + foreach ($need_launch as $repository) { + $name = $repository->getName(); + $callsign = $repository->getCallsign(); + $desc = "'{$name}' ({$callsign})"; + $phid = $repository->getPHID(); + + switch ($repository->getVersionControlSystem()) { + case 'git': + echo "Launching 'git pull' daemon on the {$desc} repository...\n"; + $control->launchDaemon( + 'PhabricatorRepositoryGitPullDaemon', + array( + $phid, + )); + echo "Launching discovery daemon on the {$desc} repository...\n"; + $control->launchDaemon( + 'PhabricatorRepositoryGitCommitDiscoveryDaemon', + array( + $phid, + )); + break; + case 'svn': + echo "Launching discovery daemon on the {$desc} repository...\n"; + $control->launchDaemon( + 'PhabricatorRepositorySvnCommitDiscoveryDaemon', + array( + $phid, + )); + break; + } + } + + echo "Launching CommitTask daemon...\n"; + $control->launchDaemon( + 'PhabricatorRepositoryCommitTaskDaemon', + array()); + + echo "Done.\n"; + } + break; + + case 'launch': $daemon = idx($argv, 2); if (!$daemon) { throw new Exception("Daemon name required!"); } - $pass = array_slice($argv, 3); + $pass_argv = array_slice($argv, 3); $n = 1; if (is_numeric($daemon)) { @@ -60,7 +125,7 @@ switch (isset($argv[1]) ? $argv[1] : 'help') { if (!$daemon) { throw new Exception("Daemon name required!"); } - $pass = array_slice($argv, 4); + $pass_argv = array_slice($argv, 4); } $loader = new PhutilSymbolLoader(); @@ -92,34 +157,13 @@ switch (isset($argv[1]) ? $argv[1] : 'help') { $daemon = reset($match); } - $libphutil_root = dirname(phutil_get_library_root('phutil')); - $launch_daemon = $libphutil_root.'/scripts/daemon/'; + echo "Launching {$n} x {$daemon}"; - // TODO: This should be a much better user experience. - Filesystem::assertExists($pid_dir); - Filesystem::assertIsDirectory($pid_dir); - Filesystem::assertWritable($pid_dir); - - foreach ($pass as $key => $arg) { - $pass[$key] = escapeshellarg($arg); - } - - echo "Starting {$n} x {$daemon}"; - - chdir($launch_daemon); for ($ii = 0; $ii < $n; $ii++) { - list($stdout, $stderr) = execx( - "./launch_daemon.php ". - "%s ". - "--load-phutil-library=%s ". - "--daemonize ". - "--phd=%s ". - implode(' ', $pass), - $daemon, - phutil_get_library_root('phabricator'), - $pid_dir); + $control->launchDaemon($daemon, $pass_argv); echo "."; } + echo "\n"; echo "Done.\n"; @@ -174,3 +218,33 @@ switch (isset($argv[1]) ? $argv[1] : 'help') { $err = $control->executeHelpCommand(); exit($err); } + +function phd_load_tracked_repositories_of_type($type) { + $repositories = phd_load_tracked_repositories(); + + foreach ($repositories as $key => $repository) { + if ($repository->getVersionControlSystem() != $type) { + unset($repositories[$key]); + } + } + + return $repositories; +} + +function phd_load_tracked_repositories() { + phutil_require_module( + 'phabricator', + 'applications/repository/storage/repository'); + + $repositories = id(new PhabricatorRepository())->loadAll(); + foreach ($repositories as $key => $repository) { + if (!$repository->getDetail('tracking-enabled')) { + unset($repositories[$key]); + } + } + + return $repositories; +} + + + diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index a6eff10bb5..8b017c19a5 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -68,6 +68,8 @@ phutil_register_library_map(array( 'ConduitAPIRequest' => 'applications/conduit/protocol/request', 'ConduitAPI_conduit_connect_Method' => 'applications/conduit/method/conduit/connect', 'ConduitAPI_conduit_ping_Method' => 'applications/conduit/method/conduit/ping', + 'ConduitAPI_daemon_launched_Method' => 'applications/conduit/method/daemon/launched', + 'ConduitAPI_daemon_log_Method' => 'applications/conduit/method/daemon/log', 'ConduitAPI_differential_creatediff_Method' => 'applications/conduit/method/differential/creatediff', 'ConduitAPI_differential_createrevision_Method' => 'applications/conduit/method/differential/createrevision', 'ConduitAPI_differential_find_Method' => 'applications/conduit/method/differential/find', @@ -207,6 +209,10 @@ phutil_register_library_map(array( 'PhabricatorDaemonConsoleController' => 'applications/daemon/controller/console', 'PhabricatorDaemonControl' => 'infrastructure/daemon/control', 'PhabricatorDaemonController' => 'applications/daemon/controller/base', + 'PhabricatorDaemonDAO' => 'infrastructure/daemon/storage/base', + 'PhabricatorDaemonLog' => 'infrastructure/daemon/storage/log', + 'PhabricatorDaemonLogEvent' => 'infrastructure/daemon/storage/event', + 'PhabricatorDaemonLogViewController' => 'applications/daemon/controller/logview', 'PhabricatorDaemonReference' => 'infrastructure/daemon/control/reference', 'PhabricatorDaemonTimelineConsoleController' => 'applications/daemon/controller/timeline', 'PhabricatorDaemonTimelineEventController' => 'applications/daemon/controller/timelineevent', @@ -421,6 +427,8 @@ phutil_register_library_map(array( 'CelerityResourceController' => 'AphrontController', 'ConduitAPI_conduit_connect_Method' => 'ConduitAPIMethod', 'ConduitAPI_conduit_ping_Method' => 'ConduitAPIMethod', + 'ConduitAPI_daemon_launched_Method' => 'ConduitAPIMethod', + 'ConduitAPI_daemon_log_Method' => 'ConduitAPIMethod', 'ConduitAPI_differential_creatediff_Method' => 'ConduitAPIMethod', 'ConduitAPI_differential_createrevision_Method' => 'ConduitAPIMethod', 'ConduitAPI_differential_find_Method' => 'ConduitAPIMethod', @@ -521,6 +529,10 @@ phutil_register_library_map(array( 'PhabricatorDaemon' => 'PhutilDaemon', 'PhabricatorDaemonConsoleController' => 'PhabricatorDaemonController', 'PhabricatorDaemonController' => 'PhabricatorController', + 'PhabricatorDaemonDAO' => 'PhabricatorLiskDAO', + 'PhabricatorDaemonLog' => 'PhabricatorDaemonDAO', + 'PhabricatorDaemonLogEvent' => 'PhabricatorDaemonDAO', + 'PhabricatorDaemonLogViewController' => 'PhabricatorDaemonController', 'PhabricatorDaemonTimelineConsoleController' => 'PhabricatorDaemonController', 'PhabricatorDaemonTimelineEventController' => 'PhabricatorDaemonController', 'PhabricatorDirectoryCategory' => 'PhabricatorDirectoryDAO', diff --git a/src/aphront/default/configuration/AphrontDefaultApplicationConfiguration.php b/src/aphront/default/configuration/AphrontDefaultApplicationConfiguration.php index 10d9924c40..e420ada3e8 100644 --- a/src/aphront/default/configuration/AphrontDefaultApplicationConfiguration.php +++ b/src/aphront/default/configuration/AphrontDefaultApplicationConfiguration.php @@ -209,6 +209,9 @@ class AphrontDefaultApplicationConfiguration ), '/daemon/' => array( + 'log/' => array( + '(?P\d+)/$' => 'PhabricatorDaemonLogViewController', + ), 'timeline/$' => 'PhabricatorDaemonTimelineConsoleController', 'timeline/(?P\d+)/$' => 'PhabricatorDaemonTimelineEventController', '$' => 'PhabricatorDaemonConsoleController', diff --git a/src/applications/conduit/method/daemon/launched/ConduitAPI_daemon_launched_Method.php b/src/applications/conduit/method/daemon/launched/ConduitAPI_daemon_launched_Method.php new file mode 100644 index 0000000000..fb27edbfd4 --- /dev/null +++ b/src/applications/conduit/method/daemon/launched/ConduitAPI_daemon_launched_Method.php @@ -0,0 +1,61 @@ + 'required string', + 'host' => 'required string', + 'pid' => 'required int', + 'argv' => 'required string', + ); + } + + public function defineReturnType() { + return 'string'; + } + + public function defineErrorTypes() { + return array( + ); + } + + protected function execute(ConduitAPIRequest $request) { + + $daemon_log = new PhabricatorDaemonLog(); + $daemon_log->setDaemon($request->getValue('daemon')); + $daemon_log->setHost($request->getValue('host')); + $daemon_log->setPID($request->getValue('pid')); + $daemon_log->setArgv(json_decode($request->getValue('argv'))); + + $daemon_log->save(); + + return $daemon_log->getID(); + } + +} diff --git a/src/applications/conduit/method/daemon/launched/__init__.php b/src/applications/conduit/method/daemon/launched/__init__.php new file mode 100644 index 0000000000..bcc64d112c --- /dev/null +++ b/src/applications/conduit/method/daemon/launched/__init__.php @@ -0,0 +1,13 @@ + 'required int', + 'type' => 'required string', + 'message' => 'optional string', + ); + } + + public function defineReturnType() { + return 'void'; + } + + public function defineErrorTypes() { + return array( + ); + } + + protected function execute(ConduitAPIRequest $request) { + + $daemon_event = new PhabricatorDaemonLogEvent(); + $daemon_event->setLogID($request->getValue('daemonLogID')); + $daemon_event->setLogType($request->getValue('type')); + $daemon_event->setMessage((string)$request->getValue('message')); + $daemon_event->setEpoch(time()); + + $daemon_event->save(); + + return; + } + +} diff --git a/src/applications/conduit/method/daemon/log/__init__.php b/src/applications/conduit/method/daemon/log/__init__.php new file mode 100644 index 0000000000..66af34a90a --- /dev/null +++ b/src/applications/conduit/method/daemon/log/__init__.php @@ -0,0 +1,13 @@ +loadAllWhere( + '1 = 1 ORDER BY id DESC LIMIT 15'); + + $rows = array(); + foreach ($logs as $log) { + $epoch = $log->getDateCreated(); + + $argv = $log->getArgv(); + $argv = array_map('phutil_escape_html', $argv); + $argv = implode('
', $argv); + + $rows[] = array( + phutil_escape_html($log->getDaemon()), + phutil_escape_html($log->getHost()), + $log->getPID(), + $argv, + date('M j, Y', $epoch), + date('g:i A', $epoch), + phutil_render_tag( + 'a', + array( + 'href' => '/daemon/log/'.$log->getID().'/', + 'class' => 'button small grey', + ), + 'View Log'), + ); + } + + $daemon_table = new AphrontTableView($rows); + $daemon_table->setHeaders( + array( + 'Daemon', + 'Host', + 'PID', + 'Argv', + 'Date', + 'Time', + 'View', + )); + $daemon_table->setColumnClasses( + array( + '', + '', + '', + 'wide wrap', + '', + 'right', + 'action', + )); + + $daemon_panel = new AphrontPanelView(); + $daemon_panel->setHeader('Recently Launched Daemons'); + $daemon_panel->appendChild($daemon_table); + $tasks = id(new PhabricatorWorkerTask())->loadAllWhere( 'leaseOwner IS NOT NULL'); @@ -118,6 +172,7 @@ class PhabricatorDaemonConsoleController extends PhabricatorDaemonController { return $this->buildStandardPageResponse( array( + $daemon_panel, $leased_panel, $queued_panel, $cursor_panel, diff --git a/src/applications/daemon/controller/console/__init__.php b/src/applications/daemon/controller/console/__init__.php index c07c4dd43a..6f34be9665 100644 --- a/src/applications/daemon/controller/console/__init__.php +++ b/src/applications/daemon/controller/console/__init__.php @@ -7,6 +7,7 @@ phutil_require_module('phabricator', 'applications/daemon/controller/base'); +phutil_require_module('phabricator', 'infrastructure/daemon/storage/log'); phutil_require_module('phabricator', 'infrastructure/daemon/timeline/storage/cursor'); phutil_require_module('phabricator', 'infrastructure/daemon/workers/storage/task'); phutil_require_module('phabricator', 'storage/queryfx'); diff --git a/src/applications/daemon/controller/logview/PhabricatorDaemonLogViewController.php b/src/applications/daemon/controller/logview/PhabricatorDaemonLogViewController.php new file mode 100644 index 0000000000..28f3c5477b --- /dev/null +++ b/src/applications/daemon/controller/logview/PhabricatorDaemonLogViewController.php @@ -0,0 +1,82 @@ +id = $data['id']; + } + + public function processRequest() { + + $log = id(new PhabricatorDaemonLog())->load($this->id); + if (!$log) { + return new Aphront404Response(); + } + + $events = id(new PhabricatorDaemonLogEvent())->loadAllWhere( + 'logID = %d ORDER BY id DESC LIMIT 200', + $log->getID()); + + $content = array(); + + + $rows = array(); + foreach ($events as $event) { + $rows[] = array( + phutil_escape_html($event->getLogType()), + date('M j, Y', $event->getEpoch()), + date('g:i:s A', $event->getEpoch()), + str_replace("\n", '
', phutil_escape_html($event->getMessage())), + ); + } + + $log_table = new AphrontTableView($rows); + $log_table->setHeaders( + array( + 'Type', + 'Date', + 'Time', + 'Message', + )); + $log_table->setColumnClasses( + array( + '', + '', + 'right', + 'wide wrap', + )); + + $log_panel = new AphrontPanelView(); + $log_panel->setHeader('Daemon Logs'); + $log_panel->appendChild($log_table); + + $content[] = $log_panel; + + + + return $this->buildStandardPageResponse( + $content, + array( + 'title' => 'Log', + )); + } + +} diff --git a/src/applications/daemon/controller/logview/__init__.php b/src/applications/daemon/controller/logview/__init__.php new file mode 100644 index 0000000000..710973b8b8 --- /dev/null +++ b/src/applications/daemon/controller/logview/__init__.php @@ -0,0 +1,20 @@ +loadAll(); + foreach ($repositories as $key => $repository) { + if (!$repository->getDetail('tracking-enabled')) { + unset($repositories[$key]); + } + } $commit = new PhabricatorRepositoryCommit(); $conn_r = $commit->establishConnection('r'); @@ -77,7 +82,8 @@ class DiffusionHomeController extends DiffusionController { 'href' => '/diffusion/'.$repository->getCallsign().'/', ), phutil_escape_html($repository->getName())), - $repository->getVersionControlSystem(), + PhabricatorRepositoryType::getNameForRepositoryType( + $repository->getVersionControlSystem()), idx($commit_counts, $id, 0), $commit ? DiffusionView::linkCommit( diff --git a/src/applications/diffusion/controller/home/__init__.php b/src/applications/diffusion/controller/home/__init__.php index c834c1e85e..ef82f3b762 100644 --- a/src/applications/diffusion/controller/home/__init__.php +++ b/src/applications/diffusion/controller/home/__init__.php @@ -8,6 +8,7 @@ phutil_require_module('phabricator', 'applications/diffusion/controller/base'); phutil_require_module('phabricator', 'applications/diffusion/view/base'); +phutil_require_module('phabricator', 'applications/repository/constants/repositorytype'); phutil_require_module('phabricator', 'applications/repository/storage/commit'); phutil_require_module('phabricator', 'applications/repository/storage/repository'); phutil_require_module('phabricator', 'storage/queryfx'); diff --git a/src/applications/repository/constants/repositorytype/PhabricatorRepositoryType.php b/src/applications/repository/constants/repositorytype/PhabricatorRepositoryType.php index c3ab64784f..23161d1f73 100644 --- a/src/applications/repository/constants/repositorytype/PhabricatorRepositoryType.php +++ b/src/applications/repository/constants/repositorytype/PhabricatorRepositoryType.php @@ -21,4 +21,13 @@ final class PhabricatorRepositoryType { const REPOSITORY_TYPE_GIT = 'git'; const REPOSITORY_TYPE_SVN = 'svn'; + public static function getNameForRepositoryType($type) { + static $map = array( + self::REPOSITORY_TYPE_GIT => 'Git', + self::REPOSITORY_TYPE_SVN => 'Subversion', + ); + + return idx($map, $type, 'Unknown'); + } + } diff --git a/src/applications/repository/constants/repositorytype/__init__.php b/src/applications/repository/constants/repositorytype/__init__.php index 0becebe43f..b2e5456a23 100644 --- a/src/applications/repository/constants/repositorytype/__init__.php +++ b/src/applications/repository/constants/repositorytype/__init__.php @@ -6,5 +6,7 @@ +phutil_require_module('phutil', 'utils'); + phutil_require_source('PhabricatorRepositoryType.php'); diff --git a/src/applications/repository/controller/list/PhabricatorRepositoryListController.php b/src/applications/repository/controller/list/PhabricatorRepositoryListController.php index 708ff68768..25610f3a34 100644 --- a/src/applications/repository/controller/list/PhabricatorRepositoryListController.php +++ b/src/applications/repository/controller/list/PhabricatorRepositoryListController.php @@ -26,10 +26,24 @@ class PhabricatorRepositoryListController $rows = array(); foreach ($repos as $repo) { + + if ($repo->getDetail('tracking-enabled')) { + $diffusion_link = phutil_render_tag( + 'a', + array( + 'href' => '/diffusion/'.$repo->getCallsign().'/', + ), + 'View in Diffusion'); + } else { + $diffusion_link = 'Not Tracked'; + } + $rows[] = array( phutil_escape_html($repo->getCallsign()), phutil_escape_html($repo->getName()), - $repo->getVersionControlSystem(), + PhabricatorRepositoryType::getNameForRepositoryType( + $repo->getVersionControlSystem()), + $diffusion_link, phutil_render_tag( 'a', array( @@ -53,6 +67,7 @@ class PhabricatorRepositoryListController 'Callsign', 'Repository', 'Type', + 'Diffusion', '', '' )); @@ -61,6 +76,7 @@ class PhabricatorRepositoryListController null, 'wide', null, + null, 'action', 'action', )); diff --git a/src/applications/repository/controller/list/__init__.php b/src/applications/repository/controller/list/__init__.php index 8b4bfb02da..ab241e5d4b 100644 --- a/src/applications/repository/controller/list/__init__.php +++ b/src/applications/repository/controller/list/__init__.php @@ -6,6 +6,7 @@ +phutil_require_module('phabricator', 'applications/repository/constants/repositorytype'); phutil_require_module('phabricator', 'applications/repository/controller/base'); phutil_require_module('phabricator', 'applications/repository/storage/repository'); phutil_require_module('phabricator', 'view/control/table'); diff --git a/src/infrastructure/daemon/control/PhabricatorDaemonControl.php b/src/infrastructure/daemon/control/PhabricatorDaemonControl.php index f5812498d8..550b16acec 100644 --- a/src/infrastructure/daemon/control/PhabricatorDaemonControl.php +++ b/src/infrastructure/daemon/control/PhabricatorDaemonControl.php @@ -141,12 +141,58 @@ final class PhabricatorDaemonControl { **parse-commit** __rXnnnn__ Parse a single commit. + **repository-launch-master** + Launches daemons to update and parse all tracked repositories. You + must also launch Taskmaster daemons, either on the same machine or + elsewhere. You should launch a master only one machine. For other + machines, launch a 'readonly'. + + **repository-launch-readonly** + Launches daemons to 'git pull' tracked git repositories so they + stay up to date. EOHELP ); return 1; } + public function launchDaemon($daemon, array $argv) { + $symbols = $this->loadAvailableDaemonClasses(); + $symbols = ipull($symbols, 'name', 'name'); + if (empty($symbols[$daemon])) { + throw new Exception("Daemon '{$daemon}' is not known."); + } + + $pid_dir = $this->getControlDirectory('pid'); + + $libphutil_root = dirname(phutil_get_library_root('phutil')); + $launch_daemon = $libphutil_root.'/scripts/daemon/'; + + // TODO: This should be a much better user experience. + Filesystem::assertExists($pid_dir); + Filesystem::assertIsDirectory($pid_dir); + Filesystem::assertWritable($pid_dir); + + foreach ($argv as $key => $arg) { + $argv[$key] = escapeshellarg($arg); + } + + // Side effect. :/ We're playing games here to keep 'ps' looking reasonable. + chdir($launch_daemon); + + execx( + "./launch_daemon.php ". + "%s ". + "--load-phutil-library=%s ". + "--conduit-uri=%s ". + "--daemonize ". + "--phd=%s ". + implode(' ', $argv), + $daemon, + phutil_get_library_root('phabricator'), + PhabricatorEnv::getURI('/api/'), + $pid_dir); + } protected function getControlDirectory($dir) { return PhabricatorEnv::getEnvConfig('phd.pid-directory').'/'.$dir; diff --git a/src/infrastructure/daemon/control/__init__.php b/src/infrastructure/daemon/control/__init__.php index b1c5285ecd..f9ca234d26 100644 --- a/src/infrastructure/daemon/control/__init__.php +++ b/src/infrastructure/daemon/control/__init__.php @@ -11,6 +11,8 @@ phutil_require_module('phabricator', 'infrastructure/env'); phutil_require_module('phutil', 'console'); phutil_require_module('phutil', 'filesystem'); +phutil_require_module('phutil', 'future/exec'); +phutil_require_module('phutil', 'moduleutils'); phutil_require_module('phutil', 'symbols'); phutil_require_module('phutil', 'utils'); diff --git a/src/infrastructure/daemon/storage/base/PhabricatorDaemonDAO.php b/src/infrastructure/daemon/storage/base/PhabricatorDaemonDAO.php new file mode 100644 index 0000000000..17acac13fb --- /dev/null +++ b/src/infrastructure/daemon/storage/base/PhabricatorDaemonDAO.php @@ -0,0 +1,25 @@ + false, + ) + parent::getConfiguration(); + } + +} diff --git a/src/infrastructure/daemon/storage/event/__init__.php b/src/infrastructure/daemon/storage/event/__init__.php new file mode 100644 index 0000000000..a021b4efeb --- /dev/null +++ b/src/infrastructure/daemon/storage/event/__init__.php @@ -0,0 +1,12 @@ + array( + 'argv' => self::SERIALIZATION_JSON, + ), + ) + parent::getConfiguration(); + } + +} diff --git a/src/infrastructure/daemon/storage/log/__init__.php b/src/infrastructure/daemon/storage/log/__init__.php new file mode 100644 index 0000000000..bca0523b98 --- /dev/null +++ b/src/infrastructure/daemon/storage/log/__init__.php @@ -0,0 +1,12 @@ +