phd - make stop / restart savvy to daemons without pid files

Summary: Ref T2374. This currently doesn't work in that while the daemons are stopped, they are restarted. How do I stop them for good? (If it worked I'd also give it a little pass for variable names, etc quality stuff)

Test Plan:
```
14:09:20 ~/Dropbox/code/phalanx (T2374)
~> sudo ./bin/phd restart
There are no running Phabricator daemons.
Freeing active task leases...
Freed 0 task lease(s).
Preparing to launch daemons.
NOTE: Logs will appear in '/var/tmp/phd/log/daemons.log'.

Launching daemon "PhabricatorRepositoryPullLocalDaemon".
Launching daemon "PhabricatorGarbageCollectorDaemon".
Launching daemon "PhabricatorTaskmasterDaemon".
Launching daemon "PhabricatorTaskmasterDaemon".
Launching daemon "PhabricatorTaskmasterDaemon".
Launching daemon "PhabricatorTaskmasterDaemon".
Done.
14:09:30 ~/Dropbox/code/phalanx (T2374)
~> rm -rf /var/tmp/phd/pid/*
14:09:42 ~/Dropbox/code/phalanx (T2374)
~> sudo ./bin/phd stop
There are no running Phabricator daemons.
There are processes running that look like Phabricator daemons but have no corresponding PID files:

php /Users/btrahan/Dropbox/code/libphutil/scripts/daemon/exec/exec_daemon.php PhabricatorRepositoryPullLocalDaemon --load-phutil-library=/Users/btrahan/Dropbox/code/arcanist/src --load-phutil-library=/Users/btrahan/Dropbox/code/phalanx/src --log=/var/tmp/phd/log/daemons.log --
php /Users/btrahan/Dropbox/code/libphutil/scripts/daemon/exec/exec_daemon.php PhabricatorGarbageCollectorDaemon --load-phutil-library=/Users/btrahan/Dropbox/code/arcanist/src --load-phutil-library=/Users/btrahan/Dropbox/code/phalanx/src --log=/var/tmp/phd/log/daemons.log --
php /Users/btrahan/Dropbox/code/libphutil/scripts/daemon/exec/exec_daemon.php PhabricatorTaskmasterDaemon --load-phutil-library=/Users/btrahan/Dropbox/code/arcanist/src --load-phutil-library=/Users/btrahan/Dropbox/code/phalanx/src --log=/var/tmp/phd/log/daemons.log --
php /Users/btrahan/Dropbox/code/libphutil/scripts/daemon/exec/exec_daemon.php PhabricatorTaskmasterDaemon --load-phutil-library=/Users/btrahan/Dropbox/code/arcanist/src --load-phutil-library=/Users/btrahan/Dropbox/code/phalanx/src --log=/var/tmp/phd/log/daemons.log --
php /Users/btrahan/Dropbox/code/libphutil/scripts/daemon/exec/exec_daemon.php PhabricatorTaskmasterDaemon --load-phutil-library=/Users/btrahan/Dropbox/code/arcanist/src --load-phutil-library=/Users/btrahan/Dropbox/code/phalanx/src --log=/var/tmp/phd/log/daemons.log --
php /Users/btrahan/Dropbox/code/libphutil/scripts/daemon/exec/exec_daemon.php PhabricatorTaskmasterDaemon --load-phutil-library=/Users/btrahan/Dropbox/code/arcanist/src --load-phutil-library=/Users/btrahan/Dropbox/code/phalanx/src --log=/var/tmp/phd/log/daemons.log --
php /Users/btrahan/Dropbox/code/phalanx/scripts/daemon/phd-daemon PhabricatorRepositoryPullLocalDaemon --daemonize --log=/var/tmp/phd/log/daemons.log --phd=/var/tmp/phd/pid
php /Users/btrahan/Dropbox/code/phalanx/scripts/daemon/phd-daemon PhabricatorGarbageCollectorDaemon --daemonize --log=/var/tmp/phd/log/daemons.log --phd=/var/tmp/phd/pid
php /Users/btrahan/Dropbox/code/phalanx/scripts/daemon/phd-daemon PhabricatorTaskmasterDaemon --daemonize --log=/var/tmp/phd/log/daemons.log --phd=/var/tmp/phd/pid
php /Users/btrahan/Dropbox/code/phalanx/scripts/daemon/phd-daemon PhabricatorTaskmasterDaemon --daemonize --log=/var/tmp/phd/log/daemons.log --phd=/var/tmp/phd/pid
php /Users/btrahan/Dropbox/code/phalanx/scripts/daemon/phd-daemon PhabricatorTaskmasterDaemon --daemonize --log=/var/tmp/phd/log/daemons.log --phd=/var/tmp/phd/pid
php /Users/btrahan/Dropbox/code/phalanx/scripts/daemon/phd-daemon PhabricatorTaskmasterDaemon --daemonize --log=/var/tmp/phd/log/daemons.log --phd=/var/tmp/phd/pid

Stop these processes by re-running this command with the --force parameter.
14:09:47 ~/Dropbox/code/phalanx (T2374)
~> sudo ./bin/phd stop --force
Interrupting daemon 'Rogue daemon' (66167)...
Interrupting daemon 'Rogue daemon' (66174)...
Interrupting daemon 'Rogue daemon' (66177)...
Interrupting daemon 'Rogue daemon' (66191)...
Interrupting daemon 'Rogue daemon' (66193)...
Interrupting daemon 'Rogue daemon' (66196)...
Interrupting daemon 'Rogue overseer' (66166)...
Interrupting daemon 'Rogue overseer' (66169)...
Interrupting daemon 'Rogue overseer' (66175)...
Interrupting daemon 'Rogue overseer' (66189)...
Interrupting daemon 'Rogue overseer' (66192)...
Interrupting daemon 'Rogue overseer' (66195)...
Daemon 66167 exited.
Daemon 66174 exited.
Daemon 66177 exited.
Daemon 66191 exited.
Daemon 66193 exited.
Daemon 66196 exited.
Daemon 66166 exited.
Daemon 66169 exited.
Daemon 66175 exited.
Daemon 66189 exited.
Daemon 66192 exited.
Daemon 66195 exited.
```

Reviewers: epriestley

Reviewed By: epriestley

Subscribers: epriestley, Korvin

Maniphest Tasks: T2374

Differential Revision: https://secure.phabricator.com/D10354

src/applications/daemon/management/PhabricatorDaemonManagementRestartWorkflow.php

@@ -19,12 +19,19 @@ final class PhabricatorDaemonManagementRestartWorkflow
               'seconds. Defaults to __15__ seconds.'),
             'default' => 15,
           ),
+          array(
+            'name' => 'force',
+            'help' => pht(
+              'Also stop running processes that look like daemons but do '.
+              'not have corresponding PID files.'),
+          ),
         ));
   }
 
   public function execute(PhutilArgumentParser $args) {
     $graceful = $args->getArg('graceful');
-    $err = $this->executeStopCommand(array(), $graceful);
+    $force = $args->getArg('force');
+    $err = $this->executeStopCommand(array(), $graceful, $force);
     if ($err) {
       return $err;
     }

src/applications/daemon/management/PhabricatorDaemonManagementStopWorkflow.php

@@ -20,6 +20,12 @@ final class PhabricatorDaemonManagementStopWorkflow
               'seconds. Defaults to __15__ seconds.'),
             'default' => 15,
           ),
+          array(
+            'name' => 'force',
+            'help' => pht(
+              'Also stop running processes that look like daemons but do '.
+              'not have corresponding PID files.'),
+          ),
           array(
             'name' => 'pids',
             'wildcard' => true,
@@ -30,7 +36,8 @@ final class PhabricatorDaemonManagementStopWorkflow
   public function execute(PhutilArgumentParser $args) {
     $pids = $args->getArg('pids');
     $graceful = $args->getArg('graceful');
-    return $this->executeStopCommand($pids, $graceful);
+    $force = $args->getArg('force');
+    return $this->executeStopCommand($pids, $graceful, $force);
   }
 
 }

src/applications/daemon/management/PhabricatorDaemonManagementWorkflow.php

@@ -286,15 +286,39 @@ abstract class PhabricatorDaemonManagementWorkflow
     return 0;
   }
 
-  protected final function executeStopCommand(array $pids, $grace_period) {
+  protected final function executeStopCommand(
+    array $pids,
+    $grace_period,
+    $force) {
+
     $console = PhutilConsole::getConsole();
 
     $daemons = $this->loadRunningDaemons();
+    $rogue_daemons = PhutilDaemonOverseer::findRunningDaemons();
     if (!$daemons) {
-      $console->writeErr(pht('There are no running Phabricator daemons.')."\n");
+      if ($force && $rogue_daemons) {
+        $stop_rogue_daemons = $this->buildRogueDaemons($rogue_daemons);
+        $this->sendStopSignals($stop_rogue_daemons, $grace_period);
+      } else {
+        $console->writeErr(pht(
+          'There are no running Phabricator daemons.')."\n");
+        if ($rogue_daemons) {
+          $console->writeErr($this->getForceStopHint($rogue_daemons)."\n");
+        }
+      }
       return 0;
     }
 
+    if ($rogue_daemons) {
+      if ($force) {
+        $daemons = array_merge(
+          $daemons,
+          $this->buildRogueDaemons($rogue_daemons));
+      } else {
+        $console->writeErr($this->getForceStopHint($rogue_daemons)."\n");
+      }
+    }
+
     $daemons = mpull($daemons, null, 'getPID');
 
     $running = array();
@@ -325,21 +349,7 @@ abstract class PhabricatorDaemonManagementWorkflow
     }
 
     $all_daemons = $running;
-
-    // If we're doing a graceful shutdown, try SIGINT first.
-    if ($grace_period) {
-      $running = $this->sendSignal($running, SIGINT, $grace_period);
-    }
-
-    // If we still have daemons, SIGTERM them.
-    if ($running) {
-      $running = $this->sendSignal($running, SIGTERM, 15);
-    }
-
-    // If the overseer is still alive, SIGKILL it.
-    if ($running) {
-      $this->sendSignal($running, SIGKILL, 0);
-    }
+    $this->sendStopSignals($running, $grace_period);
 
     foreach ($all_daemons as $daemon) {
       if ($daemon->getPIDFile()) {
@@ -350,6 +360,41 @@ abstract class PhabricatorDaemonManagementWorkflow
     return 0;
   }
 
+  private function getForceStopHint($rogue_daemons) {
+    return pht(
+      'There are processes running that look like Phabricator daemons but '.
+      'have no corresponding PID files:'."\n\n".'%s'."\n\n".
+      'Stop these processes by re-running this command with the --force '.
+      'parameter.',
+      implode("\n", ipull($rogue_daemons, 'command')));
+  }
+
+  private function buildRogueDaemons(array $daemons) {
+    $rogue_daemons = array();
+    foreach ($daemons as $pid => $data) {
+      $rogue_daemons[] =
+        PhabricatorDaemonReference::newFromRogueDictionary($data);
+    }
+    return $rogue_daemons;
+  }
+
+  private function sendStopSignals($daemons, $grace_period) {
+    // If we're doing a graceful shutdown, try SIGINT first.
+    if ($grace_period) {
+      $daemons = $this->sendSignal($daemons, SIGINT, $grace_period);
+    }
+
+    // If we still have daemons, SIGTERM them.
+    if ($daemons) {
+      $daemons = $this->sendSignal($daemons, SIGTERM, 15);
+    }
+
+    // If the overseer is still alive, SIGKILL it.
+    if ($daemons) {
+      $this->sendSignal($daemons, SIGKILL, 0);
+    }
+  }
+
   private function sendSignal(array $daemons, $signo, $wait) {
     $console = PhutilConsole::getConsole();
 

src/infrastructure/daemon/control/PhabricatorDaemonReference.php

@@ -41,6 +41,21 @@ final class PhabricatorDaemonReference {
     return $ref;
   }
 
+  /**
+   * Appropriate for getting @{class:PhabricatorDaemonReference} objects from
+   * the data from @{class:PhabricatorDaemonManagementWorkflow}'s method
+   * @{method:findRunningDaemons}.
+   *
+   * NOTE: the objects are not fully featured and should be used with caution.
+   */
+  public static function newFromRogueDictionary(array $dict) {
+    $ref = new PhabricatorDaemonReference();
+    $ref->name = pht('Rogue %s', idx($dict, 'type'));
+    $ref->pid = idx($dict, 'pid');
+
+    return $ref;
+  }
+
   public function updateStatus($new_status) {
     try {
       if (!$this->daemonLog) {