Daemons - handle daemons that can't be killed a bit better

Summary:
Ref T2374. Fixes T5988.

Keep track of what's been killed and not been killed, and surface that maybe you need sudo if things don't get killed with --force

...also basically make this force thing work. I managed to convinced myself stuff was getting killed with --force when it mostly wasn't. Make sure the --force parameter gets pushed as low as it needs to go to have things get killed.

Test Plan:
 - `sudo ./bin/phd restart`
 - `rm -rf /var/tmp/phd/pid/*`
 - `./bin/phd stop` --> get warning about rogue daemons
 - `./bin/phd stop X` --> get warning about no running daemons
 - `./bin/phd stop --force` --> get warning about not being able to kill daemons
 - `sudo ./bin/phd stop --force` --> kill daemons successfully

Reviewers: epriestley

Reviewed By: epriestley

Subscribers: epriestley, Korvin

Maniphest Tasks: T2374, T5988

Differential Revision: https://secure.phabricator.com/D10386

src/applications/daemon/management/PhabricatorDaemonManagementWorkflow.php

@@ -295,16 +295,16 @@ abstract class PhabricatorDaemonManagementWorkflow
 
     $daemons = $this->loadRunningDaemons();
     if (!$daemons) {
-      $rogue_daemons = PhutilDaemonOverseer::findRunningDaemons();
+      $survivors = array();
-      if ($force && $rogue_daemons) {
+      if (!$pids) {
-        $stop_rogue_daemons = $this->buildRogueDaemons($rogue_daemons);
+        $survivors = $this->processRogueDaemons(
-        $this->sendStopSignals($stop_rogue_daemons, $grace_period);
+          $grace_period,
-      } else {
+          $warn = true,
+          $force);
+      }
+      if (!$survivors) {
         $console->writeErr(pht(
           'There are no running Phabricator daemons.')."\n");
-        if ($rogue_daemons && !$pids) {
-          $console->writeErr($this->getForceStopHint($rogue_daemons)."\n");
-        }
       }
       return 0;
     }
@@ -339,6 +339,7 @@ abstract class PhabricatorDaemonManagementWorkflow
     }
 
     $all_daemons = $running;
+    // don't specify force here as that's about rogue daemons
     $this->sendStopSignals($running, $grace_period);
 
     foreach ($all_daemons as $daemon) {
@@ -347,17 +348,32 @@ abstract class PhabricatorDaemonManagementWorkflow
       }
     }
 
+    $this->processRogueDaemons($grace_period, !$pids, $force);
+
+    return 0;
+  }
+
+  private function processRogueDaemons($grace_period, $warn, $force_stop) {
+    $console = PhutilConsole::getConsole();
+
     $rogue_daemons = PhutilDaemonOverseer::findRunningDaemons();
     if ($rogue_daemons) {
-      if ($force) {
+      if ($force_stop) {
         $stop_rogue_daemons = $this->buildRogueDaemons($rogue_daemons);
-        $this->sendStopSignals($stop_rogue_daemons, $grace_period);
+        $survivors = $this->sendStopSignals(
-      } else if (!$pids) {
+          $stop_rogue_daemons,
+          $grace_period,
+          $force_stop);
+        if ($survivors) {
+          $console->writeErr(pht(
+            'Unable to stop processes running without pid files. Try running '.
+            'this command again with sudo.'."\n"));
+        }
+      } else if ($warn) {
         $console->writeErr($this->getForceStopHint($rogue_daemons)."\n");
       }
     }
-
+    return $rogue_daemons;
-    return 0;
   }
 
   private function getForceStopHint($rogue_daemons) {
@@ -382,31 +398,32 @@ abstract class PhabricatorDaemonManagementWorkflow
     return $rogue_daemons;
   }
 
-  private function sendStopSignals($daemons, $grace_period) {
+  private function sendStopSignals($daemons, $grace_period, $force = false) {
     // If we're doing a graceful shutdown, try SIGINT first.
     if ($grace_period) {
-      $daemons = $this->sendSignal($daemons, SIGINT, $grace_period);
+      $daemons = $this->sendSignal($daemons, SIGINT, $grace_period, $force);
     }
 
     // If we still have daemons, SIGTERM them.
     if ($daemons) {
-      $daemons = $this->sendSignal($daemons, SIGTERM, 15);
+      $daemons = $this->sendSignal($daemons, SIGTERM, 15, $force);
     }
 
     // If the overseer is still alive, SIGKILL it.
     if ($daemons) {
-      $this->sendSignal($daemons, SIGKILL, 0);
+      $daemons = $this->sendSignal($daemons, SIGKILL, 0, $force);
     }
+    return $daemons;
   }
 
-  private function sendSignal(array $daemons, $signo, $wait) {
+  private function sendSignal(array $daemons, $signo, $wait, $force = false) {
     $console = PhutilConsole::getConsole();
 
     foreach ($daemons as $key => $daemon) {
       $pid = $daemon->getPID();
       $name = $daemon->getName();
 
-      if (!$pid) {
+      if (!$pid && !$force) {
         $console->writeOut("%s\n", pht("Daemon '%s' has no PID!", $name));
         unset($daemons[$key]);
         continue;