From 7aab7e80af28f2faadb93cfc3b63a70cc8d59629 Mon Sep 17 00:00:00 2001
From: Mike Riley <mike@panopsis.co>
Date: Sun, 25 Feb 2018 20:12:30 +0000
Subject: [PATCH 01/39] Provide default values for table view properties which
 are `count`ed

Summary:
PHP 7.2 has changed the behavior of `count`, you must provide an array or `Countable` as a parameter, otherwise a warning is generated.  These two class members are counted during rendering, and are commonly left as null properties.

https://wiki.php.net/rfc/counting_non_countables

Test Plan: Browsed around my install and stopped seeing `count(): Parameter must be an array or an object that implements Countable at [AphrontTableView.php:153]` everywhere.

Reviewers: #blessed_reviewers, epriestley

Reviewed By: #blessed_reviewers, epriestley

Subscribers: Korvin, epriestley

Differential Revision: https://secure.phabricator.com/D19140
---
 src/view/control/AphrontTableView.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/view/control/AphrontTableView.php b/src/view/control/AphrontTableView.php
index 176d246718..3cca7f89e5 100644
--- a/src/view/control/AphrontTableView.php
+++ b/src/view/control/AphrontTableView.php
@@ -4,7 +4,7 @@ final class AphrontTableView extends AphrontView {
 
   protected $data;
   protected $headers;
-  protected $shortHeaders;
+  protected $shortHeaders = array();
   protected $rowClasses = array();
   protected $columnClasses = array();
   protected $cellClasses = array();
@@ -21,7 +21,7 @@ final class AphrontTableView extends AphrontView {
   protected $sortParam;
   protected $sortSelected;
   protected $sortReverse;
-  protected $sortValues;
+  protected $sortValues = array();
   private $deviceReadyTable;
 
   public function __construct(array $data) {

From cd4c4dc2ff9e6c096c011077bef61359657fa39f Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Thu, 22 Feb 2018 18:01:51 -0800
Subject: [PATCH 02/39] Add `bin/harbormaster write-log` to write some
 arbitrary content into a new Harbormaster log

Summary: Ref T13088. This is currently minimal but the modify-execute development loop on build logs is extremely long without it.

Test Plan: Ran `echo hi | ./bin/harbormaster write-log --target 12345`, saw the log show up in the web UI.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19130
---
 src/__phutil_library_map__.php                |  2 +
 ...HarbormasterManagementWriteLogWorkflow.php | 62 +++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
index 4455ad139a..2d610818e5 100644
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -1311,6 +1311,7 @@ phutil_register_library_map(array(
     'HarbormasterManagementRestartWorkflow' => 'applications/harbormaster/management/HarbormasterManagementRestartWorkflow.php',
     'HarbormasterManagementUpdateWorkflow' => 'applications/harbormaster/management/HarbormasterManagementUpdateWorkflow.php',
     'HarbormasterManagementWorkflow' => 'applications/harbormaster/management/HarbormasterManagementWorkflow.php',
+    'HarbormasterManagementWriteLogWorkflow' => 'applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php',
     'HarbormasterMessageType' => 'applications/harbormaster/engine/HarbormasterMessageType.php',
     'HarbormasterObject' => 'applications/harbormaster/storage/HarbormasterObject.php',
     'HarbormasterOtherBuildStepGroup' => 'applications/harbormaster/stepgroup/HarbormasterOtherBuildStepGroup.php',
@@ -6614,6 +6615,7 @@ phutil_register_library_map(array(
     'HarbormasterManagementRestartWorkflow' => 'HarbormasterManagementWorkflow',
     'HarbormasterManagementUpdateWorkflow' => 'HarbormasterManagementWorkflow',
     'HarbormasterManagementWorkflow' => 'PhabricatorManagementWorkflow',
+    'HarbormasterManagementWriteLogWorkflow' => 'HarbormasterManagementWorkflow',
     'HarbormasterMessageType' => 'Phobject',
     'HarbormasterObject' => 'HarbormasterDAO',
     'HarbormasterOtherBuildStepGroup' => 'HarbormasterBuildStepGroup',
diff --git a/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php b/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
new file mode 100644
index 0000000000..d84a9ed510
--- /dev/null
+++ b/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
@@ -0,0 +1,62 @@
+<?php
+
+final class HarbormasterManagementWriteLogWorkflow
+  extends HarbormasterManagementWorkflow {
+
+  protected function didConstruct() {
+    $this
+      ->setName('write-log')
+      ->setExamples('**write-log** --target __id__ [__options__]')
+      ->setSynopsis(pht('Write a new Harbormaster build log.'))
+      ->setArguments(
+        array(
+          array(
+            'name' => 'target',
+            'param' => 'id',
+            'help' => pht(
+              'Build Target ID to attach the log to.'),
+          ),
+        ));
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $viewer = $this->getViewer();
+
+    $target_id = $args->getArg('target');
+    if (!$target_id) {
+      throw new PhutilArgumentUsageException(
+        pht('Choose a build target to attach the log to with "--target".'));
+    }
+
+    $target = id(new HarbormasterBuildTargetQuery())
+      ->setViewer($viewer)
+      ->withIDs(array($target_id))
+      ->executeOne();
+    if (!$target) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'Unable to load build target "%s".',
+          $target_id));
+    }
+
+
+    $log = HarbormasterBuildLog::initializeNewBuildLog($target);
+    $log->openBuildLog();
+
+    echo tsprintf(
+      "%s\n",
+      pht('Reading log from stdin...'));
+
+    $content = file_get_contents('php://stdin');
+    $log->append($content);
+
+    $log->closeBuildLog();
+
+    echo tsprintf(
+      "%s\n",
+      pht('Done.'));
+
+    return 0;
+  }
+
+}

From 32c6b649dd211f3b2316750371c27dffe7342631 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Thu, 22 Feb 2018 18:14:59 -0800
Subject: [PATCH 03/39] Move Harbormaster log compression to the worker task
 queue

Summary: Depends on D19130. Ref T13088. Currently, when a build log is closed we compress it in the same process. Separate this out into a dedicated worker since the plan is to do a lot more work during finalization, none of which needs to happen inline during builds (or, particuarly, inline during a Conduit call for API writes in the future).

Test Plan: Ran `bin/harbormaster write-log --trace`, saw compression run inline.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19131
---
 src/__phutil_library_map__.php                |  2 +
 ...HarbormasterManagementWriteLogWorkflow.php |  7 ++-
 .../storage/build/HarbormasterBuildLog.php    | 19 ++++---
 .../worker/HarbormasterLogWorker.php          | 50 +++++++++++++++++++
 4 files changed, 71 insertions(+), 7 deletions(-)
 create mode 100644 src/applications/harbormaster/worker/HarbormasterLogWorker.php

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
index 2d610818e5..d43f386009 100644
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -1306,6 +1306,7 @@ phutil_register_library_map(array(
     'HarbormasterLeaseWorkingCopyBuildStepImplementation' => 'applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php',
     'HarbormasterLintMessagesController' => 'applications/harbormaster/controller/HarbormasterLintMessagesController.php',
     'HarbormasterLintPropertyView' => 'applications/harbormaster/view/HarbormasterLintPropertyView.php',
+    'HarbormasterLogWorker' => 'applications/harbormaster/worker/HarbormasterLogWorker.php',
     'HarbormasterManagementArchiveLogsWorkflow' => 'applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php',
     'HarbormasterManagementBuildWorkflow' => 'applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php',
     'HarbormasterManagementRestartWorkflow' => 'applications/harbormaster/management/HarbormasterManagementRestartWorkflow.php',
@@ -6610,6 +6611,7 @@ phutil_register_library_map(array(
     'HarbormasterLeaseWorkingCopyBuildStepImplementation' => 'HarbormasterBuildStepImplementation',
     'HarbormasterLintMessagesController' => 'HarbormasterController',
     'HarbormasterLintPropertyView' => 'AphrontView',
+    'HarbormasterLogWorker' => 'HarbormasterWorker',
     'HarbormasterManagementArchiveLogsWorkflow' => 'HarbormasterManagementWorkflow',
     'HarbormasterManagementBuildWorkflow' => 'HarbormasterManagementWorkflow',
     'HarbormasterManagementRestartWorkflow' => 'HarbormasterManagementWorkflow',
diff --git a/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php b/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
index d84a9ed510..b20204670b 100644
--- a/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
+++ b/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
@@ -39,7 +39,6 @@ final class HarbormasterManagementWriteLogWorkflow
           $target_id));
     }
 
-
     $log = HarbormasterBuildLog::initializeNewBuildLog($target);
     $log->openBuildLog();
 
@@ -50,6 +49,12 @@ final class HarbormasterManagementWriteLogWorkflow
     $content = file_get_contents('php://stdin');
     $log->append($content);
 
+    echo tsprintf(
+      "%s\n",
+      pht('Write completed. Closing log...'));
+
+    PhabricatorWorker::setRunAllTasksInProcess(true);
+
     $log->closeBuildLog();
 
     echo tsprintf(
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
index 290b288c6c..2826f445de 100644
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
@@ -52,17 +52,24 @@ final class HarbormasterBuildLog
       throw new Exception(pht('This build log is not open!'));
     }
 
-    if ($this->canCompressLog()) {
-      $this->compressLog();
-    }
-
     $start = $this->getDateCreated();
     $now = PhabricatorTime::getNow();
 
-    return $this
+    $this
       ->setDuration($now - $start)
       ->setLive(0)
       ->save();
+
+    PhabricatorWorker::scheduleTask(
+      'HarbormasterLogWorker',
+      array(
+        'logPHID' => $this->getPHID(),
+      ),
+      array(
+        'objectPHID' => $this->getPHID(),
+      ));
+
+    return $this;
   }
 
 
@@ -201,7 +208,7 @@ final class HarbormasterBuildLog
     return implode('', $full_text);
   }
 
-  private function canCompressLog() {
+  public function canCompressLog() {
     return function_exists('gzdeflate');
   }
 
diff --git a/src/applications/harbormaster/worker/HarbormasterLogWorker.php b/src/applications/harbormaster/worker/HarbormasterLogWorker.php
new file mode 100644
index 0000000000..a55420bc2f
--- /dev/null
+++ b/src/applications/harbormaster/worker/HarbormasterLogWorker.php
@@ -0,0 +1,50 @@
+<?php
+
+final class HarbormasterLogWorker extends HarbormasterWorker {
+
+  protected function doWork() {
+    $viewer = $this->getViewer();
+
+    $data = $this->getTaskData();
+    $log_phid = idx($data, 'logPHID');
+
+    $log = id(new HarbormasterBuildLogQuery())
+      ->setViewer($viewer)
+      ->withPHIDs(array($log_phid))
+      ->executeOne();
+    if (!$log) {
+      throw new PhabricatorWorkerPermanentFailureException(
+        pht('Invalid build log PHID "%s".', $log_phid));
+    }
+
+    $phid_key = PhabricatorHash::digestToLength($log_phid, 14);
+    $lock_key = "build.log({$phid_key})";
+    $lock = PhabricatorGlobalLock::newLock($lock_key);
+
+    try {
+      $lock->lock();
+    } catch (PhutilLockException $ex) {
+      throw new PhabricatorWorkerYieldException(15);
+    }
+
+    $caught = null;
+    try {
+      $this->finalizeBuildLog($log);
+    } catch (Exception $ex) {
+      $caught = $ex;
+    }
+
+    $lock->unlock();
+
+    if ($caught) {
+      throw $caught;
+    }
+  }
+
+  private function finalizeBuildLog(HarbormasterBuildLog $log) {
+    if ($log->canCompressLog()) {
+      $log->compressLog();
+    }
+  }
+
+}

From 8a2604cf06a8f3d2178c1311b8aa25917ad27a1d Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Thu, 22 Feb 2018 18:22:59 -0800
Subject: [PATCH 04/39] Add a "filePHID" to HarbormasterBuildLog and copy logs
 into Files during finalization

Summary: Depends on D19131. Ref T13088. During log finalization, stream the log into Files to support "Download Log", archive to Files, and API access.

Test Plan: Ran `write-log` and `rebuild-log`, saw Files objects generate with log content and appropriate permissions.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19132
---
 .../autopatches/20180222.log.01.filephid.sql  |  2 +
 src/__phutil_library_map__.php                |  2 +
 ...rbormasterManagementRebuildLogWorkflow.php | 54 ++++++++++++++
 ...HarbormasterManagementWriteLogWorkflow.php |  8 ++-
 .../phid/HarbormasterBuildLogPHIDType.php     |  2 +
 .../storage/build/HarbormasterBuildLog.php    | 26 ++++---
 .../HarbormasterBuildLogChunkIterator.php     | 12 +++-
 .../worker/HarbormasterLogWorker.php          | 70 ++++++++++++++++---
 8 files changed, 153 insertions(+), 23 deletions(-)
 create mode 100644 resources/sql/autopatches/20180222.log.01.filephid.sql
 create mode 100644 src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php

diff --git a/resources/sql/autopatches/20180222.log.01.filephid.sql b/resources/sql/autopatches/20180222.log.01.filephid.sql
new file mode 100644
index 0000000000..a7ef2f2b3e
--- /dev/null
+++ b/resources/sql/autopatches/20180222.log.01.filephid.sql
@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlog
+  ADD filePHID VARBINARY(64);
diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
index d43f386009..f22f7f6730 100644
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -1309,6 +1309,7 @@ phutil_register_library_map(array(
     'HarbormasterLogWorker' => 'applications/harbormaster/worker/HarbormasterLogWorker.php',
     'HarbormasterManagementArchiveLogsWorkflow' => 'applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php',
     'HarbormasterManagementBuildWorkflow' => 'applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php',
+    'HarbormasterManagementRebuildLogWorkflow' => 'applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php',
     'HarbormasterManagementRestartWorkflow' => 'applications/harbormaster/management/HarbormasterManagementRestartWorkflow.php',
     'HarbormasterManagementUpdateWorkflow' => 'applications/harbormaster/management/HarbormasterManagementUpdateWorkflow.php',
     'HarbormasterManagementWorkflow' => 'applications/harbormaster/management/HarbormasterManagementWorkflow.php',
@@ -6614,6 +6615,7 @@ phutil_register_library_map(array(
     'HarbormasterLogWorker' => 'HarbormasterWorker',
     'HarbormasterManagementArchiveLogsWorkflow' => 'HarbormasterManagementWorkflow',
     'HarbormasterManagementBuildWorkflow' => 'HarbormasterManagementWorkflow',
+    'HarbormasterManagementRebuildLogWorkflow' => 'HarbormasterManagementWorkflow',
     'HarbormasterManagementRestartWorkflow' => 'HarbormasterManagementWorkflow',
     'HarbormasterManagementUpdateWorkflow' => 'HarbormasterManagementWorkflow',
     'HarbormasterManagementWorkflow' => 'PhabricatorManagementWorkflow',
diff --git a/src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php b/src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php
new file mode 100644
index 0000000000..8644bb4515
--- /dev/null
+++ b/src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php
@@ -0,0 +1,54 @@
+<?php
+
+final class HarbormasterManagementRebuildLogWorkflow
+  extends HarbormasterManagementWorkflow {
+
+  protected function didConstruct() {
+    $this
+      ->setName('rebuild-log')
+      ->setExamples('**rebuild-log** --id __id__ [__options__]')
+      ->setSynopsis(
+        pht(
+          'Rebuild the file and summary for a log. This is primarily '.
+          'intended to make it easier to develop new log summarizers.'))
+      ->setArguments(
+        array(
+          array(
+            'name' => 'id',
+            'param' => 'id',
+            'help' => pht('Log to rebuild.'),
+          ),
+        ));
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $viewer = $this->getViewer();
+
+    $log_id = $args->getArg('id');
+    if (!$log_id) {
+      throw new PhutilArgumentUsageException(
+        pht('Choose a build log to rebuild with "--id".'));
+    }
+
+    $log = id(new HarbormasterBuildLogQuery())
+      ->setViewer($viewer)
+      ->withIDs(array($log_id))
+      ->executeOne();
+    if (!$log) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'Unable to load build log "%s".',
+          $log_id));
+    }
+
+    PhabricatorWorker::setRunAllTasksInProcess(true);
+    $log->scheduleRebuild(true);
+
+    echo tsprintf(
+      "%s\n",
+      pht('Done.'));
+
+    return 0;
+  }
+
+}
diff --git a/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php b/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
index b20204670b..367fe4fc78 100644
--- a/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
+++ b/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
@@ -7,14 +7,16 @@ final class HarbormasterManagementWriteLogWorkflow
     $this
       ->setName('write-log')
       ->setExamples('**write-log** --target __id__ [__options__]')
-      ->setSynopsis(pht('Write a new Harbormaster build log.'))
+      ->setSynopsis(
+        pht(
+          'Write a new Harbormaster build log. This is primarily intended '.
+          'to make development and testing easier.'))
       ->setArguments(
         array(
           array(
             'name' => 'target',
             'param' => 'id',
-            'help' => pht(
-              'Build Target ID to attach the log to.'),
+            'help' => pht('Build Target ID to attach the log to.'),
           ),
         ));
   }
diff --git a/src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php b/src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php
index c0fba81c43..dd110556b1 100644
--- a/src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php
+++ b/src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php
@@ -31,6 +31,8 @@ final class HarbormasterBuildLogPHIDType extends PhabricatorPHIDType {
 
     foreach ($handles as $phid => $handle) {
       $build_log = $objects[$phid];
+
+      $handle->setName(pht('Build Log %d', $build_log->getID()));
     }
   }
 
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
index 2826f445de..ca8f0bb9d5 100644
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
@@ -9,12 +9,13 @@ final class HarbormasterBuildLog
   protected $logType;
   protected $duration;
   protected $live;
+  protected $filePHID;
 
   private $buildTarget = self::ATTACHABLE;
   private $rope;
   private $isOpen;
 
-  const CHUNK_BYTE_LIMIT = 102400;
+  const CHUNK_BYTE_LIMIT = 1048576;
 
   public function __construct() {
     $this->rope = new PhutilRope();
@@ -60,18 +61,22 @@ final class HarbormasterBuildLog
       ->setLive(0)
       ->save();
 
-    PhabricatorWorker::scheduleTask(
-      'HarbormasterLogWorker',
-      array(
-        'logPHID' => $this->getPHID(),
-      ),
-      array(
-        'objectPHID' => $this->getPHID(),
-      ));
+    $this->scheduleRebuild(false);
 
     return $this;
   }
 
+  public function scheduleRebuild($force) {
+    PhabricatorWorker::scheduleTask(
+      'HarbormasterLogWorker',
+      array(
+        'logPHID' => $this->getPHID(),
+        'force' => $force,
+      ),
+      array(
+        'objectPHID' => $this->getPHID(),
+      ));
+  }
 
   protected function getConfiguration() {
     return array(
@@ -85,6 +90,7 @@ final class HarbormasterBuildLog
         'duration' => 'uint32?',
 
         'live' => 'bool',
+        'filePHID' => 'phid?',
       ),
       self::CONFIG_KEY_SCHEMA => array(
         'key_buildtarget' => array(
@@ -180,7 +186,7 @@ final class HarbormasterBuildLog
 
   public function newChunkIterator() {
     return id(new HarbormasterBuildLogChunkIterator($this))
-      ->setPageSize(32);
+      ->setPageSize(8);
   }
 
   private function loadLastChunkInfo() {
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php
index 754248cc67..514ad7013c 100644
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php
@@ -5,6 +5,7 @@ final class HarbormasterBuildLogChunkIterator
 
   private $log;
   private $cursor;
+  private $asString;
 
   private $min = 0;
   private $max = PHP_INT_MAX;
@@ -27,6 +28,11 @@ final class HarbormasterBuildLogChunkIterator
     return $this;
   }
 
+  public function setAsString($as_string) {
+    $this->asString = $as_string;
+    return $this;
+  }
+
   protected function loadPage() {
     if ($this->cursor > $this->max) {
       return array();
@@ -43,7 +49,11 @@ final class HarbormasterBuildLogChunkIterator
       $this->cursor = last($results)->getID() + 1;
     }
 
-    return $results;
+    if ($this->asString) {
+      return mpull($results, 'getChunkDisplayText');
+    } else {
+      return $results;
+    }
   }
 
 }
diff --git a/src/applications/harbormaster/worker/HarbormasterLogWorker.php b/src/applications/harbormaster/worker/HarbormasterLogWorker.php
index a55420bc2f..8eef7a6b6e 100644
--- a/src/applications/harbormaster/worker/HarbormasterLogWorker.php
+++ b/src/applications/harbormaster/worker/HarbormasterLogWorker.php
@@ -8,15 +8,6 @@ final class HarbormasterLogWorker extends HarbormasterWorker {
     $data = $this->getTaskData();
     $log_phid = idx($data, 'logPHID');
 
-    $log = id(new HarbormasterBuildLogQuery())
-      ->setViewer($viewer)
-      ->withPHIDs(array($log_phid))
-      ->executeOne();
-    if (!$log) {
-      throw new PhabricatorWorkerPermanentFailureException(
-        pht('Invalid build log PHID "%s".', $log_phid));
-    }
-
     $phid_key = PhabricatorHash::digestToLength($log_phid, 14);
     $lock_key = "build.log({$phid_key})";
     $lock = PhabricatorGlobalLock::newLock($lock_key);
@@ -29,6 +20,25 @@ final class HarbormasterLogWorker extends HarbormasterWorker {
 
     $caught = null;
     try {
+      $log = id(new HarbormasterBuildLogQuery())
+        ->setViewer($viewer)
+        ->withPHIDs(array($log_phid))
+        ->executeOne();
+      if (!$log) {
+        throw new PhabricatorWorkerPermanentFailureException(
+          pht(
+            'Invalid build log PHID "%s".',
+            $log_phid));
+      }
+
+      if ($log->getLive()) {
+        throw new PhabricatorWorkerPermanentFailureException(
+          pht(
+            'Log "%s" is still live. Logs can not be finalized until '.
+            'they have closed.',
+            $log_phid));
+      }
+
       $this->finalizeBuildLog($log);
     } catch (Exception $ex) {
       $caught = $ex;
@@ -42,9 +52,51 @@ final class HarbormasterLogWorker extends HarbormasterWorker {
   }
 
   private function finalizeBuildLog(HarbormasterBuildLog $log) {
+    $viewer = $this->getViewer();
+
+    $data = $this->getTaskData();
+    $is_force = idx($data, 'force');
+
     if ($log->canCompressLog()) {
       $log->compressLog();
     }
+
+    if ($is_force) {
+      $file_phid = $log->getFilePHID();
+      if ($file_phid) {
+        $file = id(new PhabricatorFileQuery())
+          ->setViewer($viewer)
+          ->withPHIDs(array($file_phid))
+          ->executeOne();
+        if ($file) {
+          id(new PhabricatorDestructionEngine())
+            ->destroyObject($file);
+        }
+        $log
+          ->setFilePHID(null)
+          ->save();
+      }
+    }
+
+    if (!$log->getFilePHID()) {
+      $iterator = $log->newChunkIterator()
+        ->setAsString(true);
+
+      $source = id(new PhabricatorIteratorFileUploadSource())
+        ->setName('harbormaster-log-'.$log->getID().'.log')
+        ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE)
+        ->setMIMEType('application/octet-stream')
+        ->setIterator($iterator);
+
+      $file = $source->uploadFile();
+
+      $file->attachToObject($log->getPHID());
+
+      $log
+        ->setFilePHID($file->getPHID())
+        ->save();
+    }
+
   }
 
 }

From 9b4295ed6051aff6bcef406db3ea75b9f41e32eb Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Thu, 22 Feb 2018 19:16:45 -0800
Subject: [PATCH 05/39] Add a very basic standalone view for build logs with a
 "Download Log" button

Summary: Depends on D19132. Ref T13088. This implements an extremely skeletal dedicated log page with a more-or-less functional "Download Log" button.

Test Plan: Downloaded a recent log. Tried to download an old (un-finalized) log, couldn't. Used `bin/harbormaster write-log` to get a convenient standalone link to a log.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19133
---
 src/__phutil_library_map__.php                |  6 ++
 .../PhabricatorHarbormasterApplication.php    |  4 ++
 ...HarbormasterBuildLogDownloadController.php | 62 +++++++++++++++++++
 .../HarbormasterBuildLogViewController.php    | 44 +++++++++++++
 ...HarbormasterManagementWriteLogWorkflow.php |  7 ++-
 .../phid/HarbormasterBuildLogPHIDType.php     |  4 +-
 .../storage/build/HarbormasterBuildLog.php    |  5 ++
 .../view/HarbormasterBuildLogView.php         | 45 ++++++++++++++
 8 files changed, 175 insertions(+), 2 deletions(-)
 create mode 100644 src/applications/harbormaster/controller/HarbormasterBuildLogDownloadController.php
 create mode 100644 src/applications/harbormaster/controller/HarbormasterBuildLogViewController.php
 create mode 100644 src/applications/harbormaster/view/HarbormasterBuildLogView.php

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
index f22f7f6730..451305d7fb 100644
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -1227,8 +1227,11 @@ phutil_register_library_map(array(
     'HarbormasterBuildLog' => 'applications/harbormaster/storage/build/HarbormasterBuildLog.php',
     'HarbormasterBuildLogChunk' => 'applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php',
     'HarbormasterBuildLogChunkIterator' => 'applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php',
+    'HarbormasterBuildLogDownloadController' => 'applications/harbormaster/controller/HarbormasterBuildLogDownloadController.php',
     'HarbormasterBuildLogPHIDType' => 'applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php',
     'HarbormasterBuildLogQuery' => 'applications/harbormaster/query/HarbormasterBuildLogQuery.php',
+    'HarbormasterBuildLogView' => 'applications/harbormaster/view/HarbormasterBuildLogView.php',
+    'HarbormasterBuildLogViewController' => 'applications/harbormaster/controller/HarbormasterBuildLogViewController.php',
     'HarbormasterBuildMessage' => 'applications/harbormaster/storage/HarbormasterBuildMessage.php',
     'HarbormasterBuildMessageQuery' => 'applications/harbormaster/query/HarbormasterBuildMessageQuery.php',
     'HarbormasterBuildPHIDType' => 'applications/harbormaster/phid/HarbormasterBuildPHIDType.php',
@@ -6511,8 +6514,11 @@ phutil_register_library_map(array(
     ),
     'HarbormasterBuildLogChunk' => 'HarbormasterDAO',
     'HarbormasterBuildLogChunkIterator' => 'PhutilBufferedIterator',
+    'HarbormasterBuildLogDownloadController' => 'HarbormasterController',
     'HarbormasterBuildLogPHIDType' => 'PhabricatorPHIDType',
     'HarbormasterBuildLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
+    'HarbormasterBuildLogView' => 'AphrontView',
+    'HarbormasterBuildLogViewController' => 'HarbormasterController',
     'HarbormasterBuildMessage' => array(
       'HarbormasterDAO',
       'PhabricatorPolicyInterface',
diff --git a/src/applications/harbormaster/application/PhabricatorHarbormasterApplication.php b/src/applications/harbormaster/application/PhabricatorHarbormasterApplication.php
index ed164b2d3b..b4f62d8e22 100644
--- a/src/applications/harbormaster/application/PhabricatorHarbormasterApplication.php
+++ b/src/applications/harbormaster/application/PhabricatorHarbormasterApplication.php
@@ -96,6 +96,10 @@ final class PhabricatorHarbormasterApplication extends PhabricatorApplication {
           'circleci/' => 'HarbormasterCircleCIHookController',
           'buildkite/' => 'HarbormasterBuildkiteHookController',
         ),
+        'log/' => array(
+          'view/(?P<id>\d+)/' => 'HarbormasterBuildLogViewController',
+          'download/(?P<id>\d+)/' => 'HarbormasterBuildLogDownloadController',
+        ),
       ),
     );
   }
diff --git a/src/applications/harbormaster/controller/HarbormasterBuildLogDownloadController.php b/src/applications/harbormaster/controller/HarbormasterBuildLogDownloadController.php
new file mode 100644
index 0000000000..6bdd04edc3
--- /dev/null
+++ b/src/applications/harbormaster/controller/HarbormasterBuildLogDownloadController.php
@@ -0,0 +1,62 @@
+<?php
+
+final class HarbormasterBuildLogDownloadController
+  extends HarbormasterController {
+
+  public function handleRequest(AphrontRequest $request) {
+    $request = $this->getRequest();
+    $viewer = $request->getUser();
+
+    $id = $request->getURIData('id');
+
+    $log = id(new HarbormasterBuildLogQuery())
+      ->setViewer($viewer)
+      ->withIDs(array($id))
+      ->executeOne();
+    if (!$log) {
+      return new Aphront404Response();
+    }
+
+    $cancel_uri = $log->getURI();
+    $file_phid = $log->getFilePHID();
+
+    if (!$file_phid) {
+      return $this->newDialog()
+        ->setTitle(pht('Log Not Finalized'))
+        ->appendParagraph(
+          pht(
+            'Logs must be fully written and processed before they can be '.
+            'downloaded. This log is still being written or processed.'))
+        ->addCancelButton($cancel_uri, pht('Wait Patiently'));
+    }
+
+    $file = id(new PhabricatorFileQuery())
+      ->setViewer($viewer)
+      ->withPHIDs(array($file_phid))
+      ->executeOne();
+    if (!$file) {
+      return $this->newDialog()
+        ->setTitle(pht('Unable to Load File'))
+        ->appendParagraph(
+          pht(
+            'Unable to load the file for this log. The file may have been '.
+            'destroyed.'))
+        ->addCancelButton($cancel_uri);
+    }
+
+    $size = $file->getByteSize();
+
+    return $this->newDialog()
+      ->setTitle(pht('Download Build Log'))
+      ->appendParagraph(
+        pht(
+          'This log has a total size of %s. If you insist, you may '.
+          'download it.',
+          phutil_tag('strong', array(), phutil_format_bytes($size))))
+      ->setDisableWorkflowOnSubmit(true)
+      ->addSubmitButton(pht('Download Log'))
+      ->setSubmitURI($file->getDownloadURI())
+      ->addCancelButton($cancel_uri, pht('Done'));
+  }
+
+}
diff --git a/src/applications/harbormaster/controller/HarbormasterBuildLogViewController.php b/src/applications/harbormaster/controller/HarbormasterBuildLogViewController.php
new file mode 100644
index 0000000000..084c48d1f1
--- /dev/null
+++ b/src/applications/harbormaster/controller/HarbormasterBuildLogViewController.php
@@ -0,0 +1,44 @@
+<?php
+
+final class HarbormasterBuildLogViewController
+  extends HarbormasterController {
+
+  public function handleRequest(AphrontRequest $request) {
+    $request = $this->getRequest();
+    $viewer = $request->getUser();
+
+    $id = $request->getURIData('id');
+
+    $log = id(new HarbormasterBuildLogQuery())
+      ->setViewer($viewer)
+      ->withIDs(array($id))
+      ->executeOne();
+    if (!$log) {
+      return new Aphront404Response();
+    }
+
+    $page_title = pht('Build Log %d', $log->getID());
+
+    $log_view = id(new HarbormasterBuildLogView())
+      ->setViewer($viewer)
+      ->setBuildLog($log);
+
+    $crumbs = $this->buildApplicationCrumbs()
+      ->addTextCrumb(pht('Build Logs'))
+      ->addTextCrumb($page_title)
+      ->setBorder(true);
+
+    $page_header = id(new PHUIHeaderView())
+      ->setHeader($page_title);
+
+    $page_view = id(new PHUITwoColumnView())
+      ->setHeader($page_header)
+      ->setFooter($log_view);
+
+    return $this->newPage()
+      ->setTitle($page_title)
+      ->setCrumbs($crumbs)
+      ->appendChild($page_view);
+  }
+
+}
diff --git a/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php b/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
index 367fe4fc78..b680b382aa 100644
--- a/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
+++ b/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
@@ -44,9 +44,14 @@ final class HarbormasterManagementWriteLogWorkflow
     $log = HarbormasterBuildLog::initializeNewBuildLog($target);
     $log->openBuildLog();
 
+    echo tsprintf(
+      "%s\n\n        __%s__\n\n",
+      pht('Opened a new build log:'),
+      PhabricatorEnv::getURI($log->getURI()));
+
     echo tsprintf(
       "%s\n",
-      pht('Reading log from stdin...'));
+      pht('Reading log content from stdin...'));
 
     $content = file_get_contents('php://stdin');
     $log->append($content);
diff --git a/src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php b/src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php
index dd110556b1..a2fa58fe79 100644
--- a/src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php
+++ b/src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php
@@ -32,7 +32,9 @@ final class HarbormasterBuildLogPHIDType extends PhabricatorPHIDType {
     foreach ($handles as $phid => $handle) {
       $build_log = $objects[$phid];
 
-      $handle->setName(pht('Build Log %d', $build_log->getID()));
+      $handle
+        ->setName(pht('Build Log %d', $build_log->getID()))
+        ->setURI($build_log->getURI());
     }
   }
 
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
index ca8f0bb9d5..17249313eb 100644
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
@@ -290,6 +290,11 @@ final class HarbormasterBuildLog
       ->save();
   }
 
+  public function getURI() {
+    $id = $this->getID();
+    return "/harbormaster/log/view/{$id}/";
+  }
+
 
 /* -(  PhabricatorPolicyInterface  )----------------------------------------- */
 
diff --git a/src/applications/harbormaster/view/HarbormasterBuildLogView.php b/src/applications/harbormaster/view/HarbormasterBuildLogView.php
new file mode 100644
index 0000000000..7543524285
--- /dev/null
+++ b/src/applications/harbormaster/view/HarbormasterBuildLogView.php
@@ -0,0 +1,45 @@
+<?php
+
+final class HarbormasterBuildLogView extends AphrontView {
+
+  private $log;
+
+  public function setBuildLog(HarbormasterBuildLog $log) {
+    $this->log = $log;
+    return $this;
+  }
+
+  public function getBuildLog() {
+    return $this->log;
+  }
+
+  public function render() {
+    $viewer = $this->getViewer();
+    $log = $this->getBuildLog();
+    $id = $log->getID();
+
+    $header = id(new PHUIHeaderView())
+      ->setViewer($viewer)
+      ->setHeader(pht('Build Log %d', $id));
+
+    $download_uri = "/harbormaster/log/download/{$id}/";
+
+    $download_button = id(new PHUIButtonView())
+      ->setTag('a')
+      ->setHref($download_uri)
+      ->setIcon('fa-download')
+      ->setDisabled(!$log->getFilePHID())
+      ->setWorkflow(true)
+      ->setText(pht('Download Log'));
+
+    $header->addActionLink($download_button);
+
+    $box_view = id(new PHUIObjectBoxView())
+      ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
+      ->setHeader($header)
+      ->appendChild('...');
+
+    return $box_view;
+  }
+
+}

From e920e2b143a2d12b1bbafabcc7ec0a282e54e322 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Fri, 23 Feb 2018 05:12:57 -0800
Subject: [PATCH 06/39] Implement DestructibleInterface on BuildLog

Summary: Depends on D19133. Ref T13088. Allows build logs to be formally destroyed, cleaning up their chunks and file data.

Test Plan:
  - Used `bin/remove destroy` to destroy a log, verified chunks and files were removed.
  - Used `bin/harbormaster rebuild-log` to force a log to rebuild, verified files were destroyed and regenerated.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19134
---
 src/__phutil_library_map__.php                |  1 +
 .../storage/build/HarbormasterBuildLog.php    | 54 ++++++++++++++++++-
 .../worker/HarbormasterLogWorker.php          | 15 +-----
 3 files changed, 54 insertions(+), 16 deletions(-)

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
index 451305d7fb..be9cb258cc 100644
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -6511,6 +6511,7 @@ phutil_register_library_map(array(
     'HarbormasterBuildLog' => array(
       'HarbormasterDAO',
       'PhabricatorPolicyInterface',
+      'PhabricatorDestructibleInterface',
     ),
     'HarbormasterBuildLogChunk' => 'HarbormasterDAO',
     'HarbormasterBuildLogChunkIterator' => 'PhutilBufferedIterator',
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
index 17249313eb..8b8fac48f2 100644
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
@@ -2,7 +2,9 @@
 
 final class HarbormasterBuildLog
   extends HarbormasterDAO
-  implements PhabricatorPolicyInterface {
+  implements
+    PhabricatorPolicyInterface,
+    PhabricatorDestructibleInterface {
 
   protected $buildTargetPHID;
   protected $logSource;
@@ -317,7 +319,55 @@ final class HarbormasterBuildLog
 
   public function describeAutomaticCapability($capability) {
     return pht(
-      "Users must be able to see a build target to view it's build log.");
+      'Users must be able to see a build target to view its build log.');
+  }
+
+
+/* -(  PhabricatorDestructibleInterface  )----------------------------------- */
+
+
+  public function destroyObjectPermanently(
+    PhabricatorDestructionEngine $engine) {
+    $this->destroyFile($engine);
+    $this->destroyChunks();
+    $this->delete();
+  }
+
+  public function destroyFile(PhabricatorDestructionEngine $engine = null) {
+    if (!$engine) {
+      $engine = new PhabricatorDestructionEngine();
+    }
+
+    $file_phid = $this->getFilePHID();
+    if ($file_phid) {
+      $viewer = $engine->getViewer();
+      $file = id(new PhabricatorFileQuery())
+        ->setViewer($viewer)
+        ->withPHIDs(array($file_phid))
+        ->executeOne();
+      if ($file) {
+        $engine->destroyObject($file);
+      }
+    }
+
+    $this->setFilePHID(null);
+
+    return $this;
+  }
+
+  public function destroyChunks() {
+    $chunk = new HarbormasterBuildLogChunk();
+    $conn = $chunk->establishConnection('w');
+
+    // Just delete the chunks directly so we don't have to pull the data over
+    // the wire for large logs.
+    queryfx(
+      $conn,
+      'DELETE FROM %T WHERE logID = %d',
+      $chunk->getTableName(),
+      $this->getID());
+
+    return $this;
   }
 
 
diff --git a/src/applications/harbormaster/worker/HarbormasterLogWorker.php b/src/applications/harbormaster/worker/HarbormasterLogWorker.php
index 8eef7a6b6e..615781da66 100644
--- a/src/applications/harbormaster/worker/HarbormasterLogWorker.php
+++ b/src/applications/harbormaster/worker/HarbormasterLogWorker.php
@@ -62,20 +62,7 @@ final class HarbormasterLogWorker extends HarbormasterWorker {
     }
 
     if ($is_force) {
-      $file_phid = $log->getFilePHID();
-      if ($file_phid) {
-        $file = id(new PhabricatorFileQuery())
-          ->setViewer($viewer)
-          ->withPHIDs(array($file_phid))
-          ->executeOne();
-        if ($file) {
-          id(new PhabricatorDestructionEngine())
-            ->destroyObject($file);
-        }
-        $log
-          ->setFilePHID(null)
-          ->save();
-      }
+      $log->destroyFile();
     }
 
     if (!$log->getFilePHID()) {

From d152bd58360884f41105dc642225492730b536ac Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Fri, 23 Feb 2018 05:32:51 -0800
Subject: [PATCH 07/39] Manage log locks on the Log object to prepare for
 multiple writers

Summary:
Depends on D19134. Ref T13088. Future changes will support API writers, so push the log lock into the Log object.

Allow open/close ("this process is writing to this log") to be separate from live/final ("this log is still generating more data").

Test Plan: Wrote logs with `bin/harbormater write-log` and updated logs with `bin/harbormaster rebuild-log`.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19135
---
 .../storage/build/HarbormasterBuildLog.php    | 261 +++++++++++-------
 .../worker/HarbormasterLogWorker.php          |  26 +-
 2 files changed, 174 insertions(+), 113 deletions(-)

diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
index 8b8fac48f2..0ae4446837 100644
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
@@ -16,6 +16,7 @@ final class HarbormasterBuildLog
   private $buildTarget = self::ATTACHABLE;
   private $rope;
   private $isOpen;
+  private $lock;
 
   const CHUNK_BYTE_LIMIT = 1048576;
 
@@ -27,6 +28,12 @@ final class HarbormasterBuildLog
     if ($this->isOpen) {
       $this->closeBuildLog();
     }
+
+    if ($this->lock) {
+      if ($this->lock->isLocked()) {
+        $this->lock->unlock();
+      }
+    }
   }
 
   public static function initializeNewBuildLog(
@@ -35,37 +42,7 @@ final class HarbormasterBuildLog
     return id(new HarbormasterBuildLog())
       ->setBuildTargetPHID($build_target->getPHID())
       ->setDuration(null)
-      ->setLive(0);
-  }
-
-  public function openBuildLog() {
-    if ($this->isOpen) {
-      throw new Exception(pht('This build log is already open!'));
-    }
-
-    $this->isOpen = true;
-
-    return $this
-      ->setLive(1)
-      ->save();
-  }
-
-  public function closeBuildLog() {
-    if (!$this->isOpen) {
-      throw new Exception(pht('This build log is not open!'));
-    }
-
-    $start = $this->getDateCreated();
-    $now = PhabricatorTime::getNow();
-
-    $this
-      ->setDuration($now - $start)
-      ->setLive(0)
-      ->save();
-
-    $this->scheduleRebuild(false);
-
-    return $this;
+      ->setLive(1);
   }
 
   public function scheduleRebuild($force) {
@@ -120,72 +97,6 @@ final class HarbormasterBuildLog
     return pht('Build Log');
   }
 
-  public function append($content) {
-    if (!$this->getLive()) {
-      throw new PhutilInvalidStateException('openBuildLog');
-    }
-
-    $content = (string)$content;
-
-    $this->rope->append($content);
-    $this->flush();
-
-    return $this;
-  }
-
-  private function flush() {
-
-    // TODO: Maybe don't flush more than a couple of times per second. If a
-    // caller writes a single character over and over again, we'll currently
-    // spend a lot of time flushing that.
-
-    $chunk_table = id(new HarbormasterBuildLogChunk())->getTableName();
-    $chunk_limit = self::CHUNK_BYTE_LIMIT;
-    $encoding_text = HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT;
-
-    $rope = $this->rope;
-
-    while (true) {
-      $length = $rope->getByteLength();
-      if (!$length) {
-        break;
-      }
-
-      $conn_w = $this->establishConnection('w');
-      $last = $this->loadLastChunkInfo();
-
-      $can_append =
-        ($last) &&
-        ($last['encoding'] == $encoding_text) &&
-        ($last['size'] < $chunk_limit);
-      if ($can_append) {
-        $append_id = $last['id'];
-        $prefix_size = $last['size'];
-      } else {
-        $append_id = null;
-        $prefix_size = 0;
-      }
-
-      $data_limit = ($chunk_limit - $prefix_size);
-      $append_data = $rope->getPrefixBytes($data_limit);
-      $data_size = strlen($append_data);
-
-      if ($append_id) {
-        queryfx(
-          $conn_w,
-          'UPDATE %T SET chunk = CONCAT(chunk, %B), size = %d WHERE id = %d',
-          $chunk_table,
-          $append_data,
-          $prefix_size + $data_size,
-          $append_id);
-      } else {
-        $this->writeChunk($encoding_text, $data_size, $append_data);
-      }
-
-      $rope->removeBytesFromHead($data_size);
-    }
-  }
-
   public function newChunkIterator() {
     return id(new HarbormasterBuildLogChunkIterator($this))
       ->setPageSize(8);
@@ -216,6 +127,16 @@ final class HarbormasterBuildLog
     return implode('', $full_text);
   }
 
+
+  public function getURI() {
+    $id = $this->getID();
+    return "/harbormaster/log/view/{$id}/";
+  }
+
+
+/* -(  Chunks  )------------------------------------------------------------- */
+
+
   public function canCompressLog() {
     return function_exists('gzdeflate');
   }
@@ -229,6 +150,13 @@ final class HarbormasterBuildLog
   }
 
   private function processLog($mode) {
+    if (!$this->getLock()->isLocked()) {
+      throw new Exception(
+        pht(
+          'You can not process build log chunks unless the log lock is '.
+          'held.'));
+    }
+
     $chunks = $this->newChunkIterator();
 
     // NOTE: Because we're going to insert new chunks, we need to stop the
@@ -292,12 +220,145 @@ final class HarbormasterBuildLog
       ->save();
   }
 
-  public function getURI() {
-    $id = $this->getID();
-    return "/harbormaster/log/view/{$id}/";
+
+/* -(  Writing  )------------------------------------------------------------ */
+
+
+  public function getLock() {
+    if (!$this->lock) {
+      $phid = $this->getPHID();
+      $phid_key = PhabricatorHash::digestToLength($phid, 14);
+      $lock_key = "build.log({$phid_key})";
+      $lock = PhabricatorGlobalLock::newLock($lock_key);
+      $this->lock = $lock;
+    }
+
+    return $this->lock;
   }
 
 
+  public function openBuildLog() {
+    if ($this->isOpen) {
+      throw new Exception(pht('This build log is already open!'));
+    }
+
+    $is_new = !$this->getID();
+    if ($is_new) {
+      $this->save();
+    }
+
+    $this->getLock()->lock();
+    $this->isOpen = true;
+
+    $this->reload();
+
+    if (!$this->getLive()) {
+      $this->setLive(1)->save();
+    }
+
+    return $this;
+  }
+
+  public function closeBuildLog($forever = true) {
+    if (!$this->isOpen) {
+      throw new Exception(
+        pht(
+          'You must openBuildLog() before you can closeBuildLog().'));
+    }
+
+    $this->flush();
+
+    if ($forever) {
+      $start = $this->getDateCreated();
+      $now = PhabricatorTime::getNow();
+
+      $this
+        ->setDuration($now - $start)
+        ->setLive(0)
+        ->save();
+    }
+
+    $this->getLock()->unlock();
+    $this->isOpen = false;
+
+    if ($forever) {
+      $this->scheduleRebuild(false);
+    }
+
+    return $this;
+  }
+
+  public function append($content) {
+    if (!$this->isOpen) {
+      throw new Exception(
+        pht(
+          'You must openBuildLog() before you can append() content to '.
+          'the log.'));
+    }
+
+    $content = (string)$content;
+
+    $this->rope->append($content);
+    $this->flush();
+
+    return $this;
+  }
+
+  private function flush() {
+
+    // TODO: Maybe don't flush more than a couple of times per second. If a
+    // caller writes a single character over and over again, we'll currently
+    // spend a lot of time flushing that.
+
+    $chunk_table = id(new HarbormasterBuildLogChunk())->getTableName();
+    $chunk_limit = self::CHUNK_BYTE_LIMIT;
+    $encoding_text = HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT;
+
+    $rope = $this->rope;
+
+    while (true) {
+      $length = $rope->getByteLength();
+      if (!$length) {
+        break;
+      }
+
+      $conn_w = $this->establishConnection('w');
+      $last = $this->loadLastChunkInfo();
+
+      $can_append =
+        ($last) &&
+        ($last['encoding'] == $encoding_text) &&
+        ($last['size'] < $chunk_limit);
+      if ($can_append) {
+        $append_id = $last['id'];
+        $prefix_size = $last['size'];
+      } else {
+        $append_id = null;
+        $prefix_size = 0;
+      }
+
+      $data_limit = ($chunk_limit - $prefix_size);
+      $append_data = $rope->getPrefixBytes($data_limit);
+      $data_size = strlen($append_data);
+
+      if ($append_id) {
+        queryfx(
+          $conn_w,
+          'UPDATE %T SET chunk = CONCAT(chunk, %B), size = %d WHERE id = %d',
+          $chunk_table,
+          $append_data,
+          $prefix_size + $data_size,
+          $append_id);
+      } else {
+        $this->writeChunk($encoding_text, $data_size, $append_data);
+      }
+
+      $rope->removeBytesFromHead($data_size);
+    }
+  }
+
+
+
 /* -(  PhabricatorPolicyInterface  )----------------------------------------- */
 
 
diff --git a/src/applications/harbormaster/worker/HarbormasterLogWorker.php b/src/applications/harbormaster/worker/HarbormasterLogWorker.php
index 615781da66..b0c84b52f3 100644
--- a/src/applications/harbormaster/worker/HarbormasterLogWorker.php
+++ b/src/applications/harbormaster/worker/HarbormasterLogWorker.php
@@ -8,9 +8,18 @@ final class HarbormasterLogWorker extends HarbormasterWorker {
     $data = $this->getTaskData();
     $log_phid = idx($data, 'logPHID');
 
-    $phid_key = PhabricatorHash::digestToLength($log_phid, 14);
-    $lock_key = "build.log({$phid_key})";
-    $lock = PhabricatorGlobalLock::newLock($lock_key);
+    $log = id(new HarbormasterBuildLogQuery())
+      ->setViewer($viewer)
+      ->withPHIDs(array($log_phid))
+      ->executeOne();
+    if (!$log) {
+      throw new PhabricatorWorkerPermanentFailureException(
+        pht(
+          'Invalid build log PHID "%s".',
+          $log_phid));
+    }
+
+    $lock = $log->getLock();
 
     try {
       $lock->lock();
@@ -20,16 +29,7 @@ final class HarbormasterLogWorker extends HarbormasterWorker {
 
     $caught = null;
     try {
-      $log = id(new HarbormasterBuildLogQuery())
-        ->setViewer($viewer)
-        ->withPHIDs(array($log_phid))
-        ->executeOne();
-      if (!$log) {
-        throw new PhabricatorWorkerPermanentFailureException(
-          pht(
-            'Invalid build log PHID "%s".',
-            $log_phid));
-      }
+      $log->reload();
 
       if ($log->getLive()) {
         throw new PhabricatorWorkerPermanentFailureException(

From 57e3d607f566ba1cddbca5b149738632359ae80e Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Fri, 23 Feb 2018 05:48:07 -0800
Subject: [PATCH 08/39] In Harbormaster, record byte length on the build logs

Summary: Depends on D19135. Ref T13088. Denormalize the total log size onto the log itself. This makes reasoning about the log at display time easier, and we don't need to fish around in the database as much to figure out what we're dealing with.

Test Plan: Ran `bin/harbormaster rebuild-log`, saw an existing log populate. Ran `bin/harbormaster write-log`, saw new log write with proper length information.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19136
---
 .../20180223.log.01.bytelength.sql            |  2 ++
 .../storage/build/HarbormasterBuildLog.php    | 33 +++++++++++--------
 .../worker/HarbormasterLogWorker.php          | 13 ++++++++
 3 files changed, 35 insertions(+), 13 deletions(-)
 create mode 100644 resources/sql/autopatches/20180223.log.01.bytelength.sql

diff --git a/resources/sql/autopatches/20180223.log.01.bytelength.sql b/resources/sql/autopatches/20180223.log.01.bytelength.sql
new file mode 100644
index 0000000000..a4c3505628
--- /dev/null
+++ b/resources/sql/autopatches/20180223.log.01.bytelength.sql
@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlog
+  ADD byteLength BIGINT UNSIGNED NOT NULL;
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
index 0ae4446837..f43f028d5d 100644
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
@@ -12,6 +12,7 @@ final class HarbormasterBuildLog
   protected $duration;
   protected $live;
   protected $filePHID;
+  protected $byteLength;
 
   private $buildTarget = self::ATTACHABLE;
   private $rope;
@@ -42,7 +43,8 @@ final class HarbormasterBuildLog
     return id(new HarbormasterBuildLog())
       ->setBuildTargetPHID($build_target->getPHID())
       ->setDuration(null)
-      ->setLive(1);
+      ->setLive(1)
+      ->setByteLength(0);
   }
 
   public function scheduleRebuild($force) {
@@ -70,6 +72,7 @@ final class HarbormasterBuildLog
 
         'live' => 'bool',
         'filePHID' => 'phid?',
+        'byteLength' => 'uint64',
       ),
       self::CONFIG_KEY_SCHEMA => array(
         'key_buildtarget' => array(
@@ -341,24 +344,28 @@ final class HarbormasterBuildLog
       $append_data = $rope->getPrefixBytes($data_limit);
       $data_size = strlen($append_data);
 
-      if ($append_id) {
-        queryfx(
-          $conn_w,
-          'UPDATE %T SET chunk = CONCAT(chunk, %B), size = %d WHERE id = %d',
-          $chunk_table,
-          $append_data,
-          $prefix_size + $data_size,
-          $append_id);
-      } else {
-        $this->writeChunk($encoding_text, $data_size, $append_data);
-      }
+      $this->openTransaction();
+        if ($append_id) {
+          queryfx(
+            $conn_w,
+            'UPDATE %T SET chunk = CONCAT(chunk, %B), size = %d WHERE id = %d',
+            $chunk_table,
+            $append_data,
+            $prefix_size + $data_size,
+            $append_id);
+        } else {
+          $this->writeChunk($encoding_text, $data_size, $append_data);
+        }
+
+        $this->byteLength += $data_size;
+        $this->save();
+      $this->saveTransaction();
 
       $rope->removeBytesFromHead($data_size);
     }
   }
 
 
-
 /* -(  PhabricatorPolicyInterface  )----------------------------------------- */
 
 
diff --git a/src/applications/harbormaster/worker/HarbormasterLogWorker.php b/src/applications/harbormaster/worker/HarbormasterLogWorker.php
index b0c84b52f3..d90c2391c0 100644
--- a/src/applications/harbormaster/worker/HarbormasterLogWorker.php
+++ b/src/applications/harbormaster/worker/HarbormasterLogWorker.php
@@ -57,6 +57,19 @@ final class HarbormasterLogWorker extends HarbormasterWorker {
     $data = $this->getTaskData();
     $is_force = idx($data, 'force');
 
+    if (!$log->getByteLength() || $is_force) {
+      $iterator = $log->newChunkIterator()
+        ->setAsString(true);
+
+      $byte_length = 0;
+      foreach ($iterator as $block) {
+        $byte_length += strlen($block);
+      }
+      $log
+        ->setByteLength($byte_length)
+        ->save();
+    }
+
     if ($log->canCompressLog()) {
       $log->compressLog();
     }

From 46d735d312ca8e54f9b2604783b62c360bf7a1a9 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Fri, 23 Feb 2018 05:55:52 -0800
Subject: [PATCH 09/39] Add "--all" and an explicit "--force" flag to
 `bin/harbormaster rebuild-log`

Summary: Depends on D19136. Ref T13088. Since it's probably impractical to do all the migrations these changes imply during `bin/storage upgrade`, provide some support for performing them online.

Test Plan: Ran `bin/harbormaster rebuild-log` with `--all`, `--id`, and with and without `--force`.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19137
---
 ...rbormasterManagementRebuildLogWorkflow.php | 76 +++++++++++++++----
 1 file changed, 62 insertions(+), 14 deletions(-)

diff --git a/src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php b/src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php
index 8644bb4515..1dc91bcf0f 100644
--- a/src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php
+++ b/src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php
@@ -6,7 +6,10 @@ final class HarbormasterManagementRebuildLogWorkflow
   protected function didConstruct() {
     $this
       ->setName('rebuild-log')
-      ->setExamples('**rebuild-log** --id __id__ [__options__]')
+      ->setExamples(
+        pht(
+          "**rebuild-log** --id __id__ [__options__]\n".
+          "**rebuild-log** --all"))
       ->setSynopsis(
         pht(
           'Rebuild the file and summary for a log. This is primarily '.
@@ -18,31 +21,76 @@ final class HarbormasterManagementRebuildLogWorkflow
             'param' => 'id',
             'help' => pht('Log to rebuild.'),
           ),
+          array(
+            'name' => 'all',
+            'help' => pht('Rebuild all logs.'),
+          ),
+          array(
+            'name' => 'force',
+            'help' => pht(
+              'Force logs to rebuild even if they appear to be in good '.
+              'shape already.'),
+          ),
         ));
   }
 
   public function execute(PhutilArgumentParser $args) {
     $viewer = $this->getViewer();
 
-    $log_id = $args->getArg('id');
-    if (!$log_id) {
-      throw new PhutilArgumentUsageException(
-        pht('Choose a build log to rebuild with "--id".'));
-    }
+    $is_force = $args->getArg('force');
 
-    $log = id(new HarbormasterBuildLogQuery())
-      ->setViewer($viewer)
-      ->withIDs(array($log_id))
-      ->executeOne();
-    if (!$log) {
+    $log_id = $args->getArg('id');
+    $is_all = $args->getArg('all');
+
+    if (!$is_all && !$log_id) {
       throw new PhutilArgumentUsageException(
         pht(
-          'Unable to load build log "%s".',
-          $log_id));
+          'Choose a build log to rebuild with "--id", or rebuild all '.
+          'logs with "--all".'));
+    }
+
+    if ($is_all && $log_id) {
+      throw new PhutilArgumentUsageException(
+        pht(
+          'You can not specify both "--id" and "--all". Choose one or '.
+          'the other.'));
+    }
+
+    if ($log_id) {
+      $log = id(new HarbormasterBuildLogQuery())
+        ->setViewer($viewer)
+        ->withIDs(array($log_id))
+        ->executeOne();
+      if (!$log) {
+        throw new PhutilArgumentUsageException(
+          pht(
+            'Unable to load build log "%s".',
+            $log_id));
+      }
+      $logs = array($log);
+    } else {
+      $logs = new LiskMigrationIterator(new HarbormasterBuildLog());
     }
 
     PhabricatorWorker::setRunAllTasksInProcess(true);
-    $log->scheduleRebuild(true);
+
+    foreach ($logs as $log) {
+      echo tsprintf(
+        "%s\n",
+        pht(
+          'Rebuilding log "%s"...',
+          pht('Build Log %d', $log->getID())));
+
+      try {
+        $log->scheduleRebuild($is_force);
+      } catch (Exception $ex) {
+        if ($is_all) {
+          phlog($ex);
+        } else {
+          throw $ex;
+        }
+      }
+    }
 
     echo tsprintf(
       "%s\n",

From d6311044bba66828f8e5d8898662d9dfe40e3088 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Fri, 23 Feb 2018 06:07:14 -0800
Subject: [PATCH 10/39] Store the Harbormaster log chunk format on the log
 record

Summary: Depends on D19137. Ref T13088. This allows `rebuild-log` to skip work if the chunks are already compressed. It also prepares for a future GC which is looking for "text" or "gzip" chunks to throw away in favor of archival into Files; such a GC can use this column to find collectable logs and then write "file" to it, meaning "chunks are gone, this data is only available in Files".

Test Plan: Ran migration, saw logs populate as "text". Ran `rebuild-log`, saw logs rebuild as "gzip".

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19138
---
 .../autopatches/20180223.log.02.chunkformat.sql    |  2 ++
 .../autopatches/20180223.log.03.chunkdefault.sql   |  2 ++
 .../storage/build/HarbormasterBuildLog.php         | 14 +++++++++++++-
 .../harbormaster/worker/HarbormasterLogWorker.php  | 14 ++++++++------
 4 files changed, 25 insertions(+), 7 deletions(-)
 create mode 100644 resources/sql/autopatches/20180223.log.02.chunkformat.sql
 create mode 100644 resources/sql/autopatches/20180223.log.03.chunkdefault.sql

diff --git a/resources/sql/autopatches/20180223.log.02.chunkformat.sql b/resources/sql/autopatches/20180223.log.02.chunkformat.sql
new file mode 100644
index 0000000000..a15676a952
--- /dev/null
+++ b/resources/sql/autopatches/20180223.log.02.chunkformat.sql
@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlog
+  ADD chunkFormat VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT};
diff --git a/resources/sql/autopatches/20180223.log.03.chunkdefault.sql b/resources/sql/autopatches/20180223.log.03.chunkdefault.sql
new file mode 100644
index 0000000000..2a1f2c812b
--- /dev/null
+++ b/resources/sql/autopatches/20180223.log.03.chunkdefault.sql
@@ -0,0 +1,2 @@
+UPDATE {$NAMESPACE}_harbormaster.harbormaster_buildlog
+  SET chunkFormat = 'text' WHERE chunkFormat = '';
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
index f43f028d5d..6b0b15d8b1 100644
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
@@ -13,6 +13,7 @@ final class HarbormasterBuildLog
   protected $live;
   protected $filePHID;
   protected $byteLength;
+  protected $chunkFormat;
 
   private $buildTarget = self::ATTACHABLE;
   private $rope;
@@ -44,7 +45,8 @@ final class HarbormasterBuildLog
       ->setBuildTargetPHID($build_target->getPHID())
       ->setDuration(null)
       ->setLive(1)
-      ->setByteLength(0);
+      ->setByteLength(0)
+      ->setChunkFormat(HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT);
   }
 
   public function scheduleRebuild($force) {
@@ -73,6 +75,7 @@ final class HarbormasterBuildLog
         'live' => 'bool',
         'filePHID' => 'phid?',
         'byteLength' => 'uint64',
+        'chunkFormat' => 'text32',
       ),
       self::CONFIG_KEY_SCHEMA => array(
         'key_buildtarget' => array(
@@ -105,6 +108,11 @@ final class HarbormasterBuildLog
       ->setPageSize(8);
   }
 
+  public function newDataIterator() {
+    return $this->newChunkIterator()
+      ->setAsString(true);
+  }
+
   private function loadLastChunkInfo() {
     $chunk_table = new HarbormasterBuildLogChunk();
     $conn_w = $chunk_table->establishConnection('w');
@@ -188,6 +196,10 @@ final class HarbormasterBuildLog
       $this->writeEncodedChunk($rope, $byte_limit, $mode);
     }
 
+    $this
+      ->setChunkFormat($mode)
+      ->save();
+
     $this->saveTransaction();
   }
 
diff --git a/src/applications/harbormaster/worker/HarbormasterLogWorker.php b/src/applications/harbormaster/worker/HarbormasterLogWorker.php
index d90c2391c0..fc8762de0a 100644
--- a/src/applications/harbormaster/worker/HarbormasterLogWorker.php
+++ b/src/applications/harbormaster/worker/HarbormasterLogWorker.php
@@ -58,20 +58,23 @@ final class HarbormasterLogWorker extends HarbormasterWorker {
     $is_force = idx($data, 'force');
 
     if (!$log->getByteLength() || $is_force) {
-      $iterator = $log->newChunkIterator()
-        ->setAsString(true);
+      $iterator = $log->newDataIterator();
 
       $byte_length = 0;
       foreach ($iterator as $block) {
         $byte_length += strlen($block);
       }
+
       $log
         ->setByteLength($byte_length)
         ->save();
     }
 
-    if ($log->canCompressLog()) {
-      $log->compressLog();
+    $format_text = HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT;
+    if (($log->getChunkFormat() === $format_text) || $is_force) {
+      if ($log->canCompressLog()) {
+        $log->compressLog();
+      }
     }
 
     if ($is_force) {
@@ -79,8 +82,7 @@ final class HarbormasterLogWorker extends HarbormasterWorker {
     }
 
     if (!$log->getFilePHID()) {
-      $iterator = $log->newChunkIterator()
-        ->setAsString(true);
+      $iterator = $log->newDataIterator();
 
       $source = id(new PhabricatorIteratorFileUploadSource())
         ->setName('harbormaster-log-'.$log->getID().'.log')

From 6dc341be87088984beb6ff4a267c4deff57a0bd5 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Fri, 23 Feb 2018 07:10:18 -0800
Subject: [PATCH 11/39] As Harbormaster logs are processed, build a sparse map
 of byte offsets to line numbers

Summary:
Depends on D19138. Ref T13088. When we want to read the last part of a logfile //and show accurate line numbers//, we need to be able to get from byte offsets to line numbers somehow.

Our fundamental unit must remain byte offsets, because a test can emit an arbitrarily long line, and we should accommodate it cleanly if a test emits 2GB of the letter "A".

To support going from byte offsets to line numbers, compute a map with periodic line markers throughout the offsets of the file. From here, we can figure out the line numbers for arbitrary positions in the file with only a constant amount of work.

Test Plan: Added unit tests; ran unit tests.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19139
---
 .../autopatches/20180223.log.04.linemap.sql   |   2 +
 .../20180223.log.05.linemapdefault.sql        |   2 +
 src/__phutil_library_map__.php                |   2 +
 .../HarbormasterBuildLogTestCase.php          | 117 ++++++++++++++++
 .../storage/build/HarbormasterBuildLog.php    | 131 +++++++++++++++++-
 .../worker/HarbormasterLogWorker.php          |  13 +-
 6 files changed, 260 insertions(+), 7 deletions(-)
 create mode 100644 resources/sql/autopatches/20180223.log.04.linemap.sql
 create mode 100644 resources/sql/autopatches/20180223.log.05.linemapdefault.sql
 create mode 100644 src/applications/harbormaster/__tests__/HarbormasterBuildLogTestCase.php

diff --git a/resources/sql/autopatches/20180223.log.04.linemap.sql b/resources/sql/autopatches/20180223.log.04.linemap.sql
new file mode 100644
index 0000000000..75ed27cf7c
--- /dev/null
+++ b/resources/sql/autopatches/20180223.log.04.linemap.sql
@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlog
+  ADD lineMap LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT};
diff --git a/resources/sql/autopatches/20180223.log.05.linemapdefault.sql b/resources/sql/autopatches/20180223.log.05.linemapdefault.sql
new file mode 100644
index 0000000000..59b4dc9a62
--- /dev/null
+++ b/resources/sql/autopatches/20180223.log.05.linemapdefault.sql
@@ -0,0 +1,2 @@
+UPDATE {$NAMESPACE}_harbormaster.harbormaster_buildlog
+  SET lineMap = '[]' WHERE lineMap = '';
diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
index be9cb258cc..01407ed67b 100644
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -1230,6 +1230,7 @@ phutil_register_library_map(array(
     'HarbormasterBuildLogDownloadController' => 'applications/harbormaster/controller/HarbormasterBuildLogDownloadController.php',
     'HarbormasterBuildLogPHIDType' => 'applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php',
     'HarbormasterBuildLogQuery' => 'applications/harbormaster/query/HarbormasterBuildLogQuery.php',
+    'HarbormasterBuildLogTestCase' => 'applications/harbormaster/__tests__/HarbormasterBuildLogTestCase.php',
     'HarbormasterBuildLogView' => 'applications/harbormaster/view/HarbormasterBuildLogView.php',
     'HarbormasterBuildLogViewController' => 'applications/harbormaster/controller/HarbormasterBuildLogViewController.php',
     'HarbormasterBuildMessage' => 'applications/harbormaster/storage/HarbormasterBuildMessage.php',
@@ -6518,6 +6519,7 @@ phutil_register_library_map(array(
     'HarbormasterBuildLogDownloadController' => 'HarbormasterController',
     'HarbormasterBuildLogPHIDType' => 'PhabricatorPHIDType',
     'HarbormasterBuildLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
+    'HarbormasterBuildLogTestCase' => 'PhabricatorTestCase',
     'HarbormasterBuildLogView' => 'AphrontView',
     'HarbormasterBuildLogViewController' => 'HarbormasterController',
     'HarbormasterBuildMessage' => array(
diff --git a/src/applications/harbormaster/__tests__/HarbormasterBuildLogTestCase.php b/src/applications/harbormaster/__tests__/HarbormasterBuildLogTestCase.php
new file mode 100644
index 0000000000..6cbdd6b0db
--- /dev/null
+++ b/src/applications/harbormaster/__tests__/HarbormasterBuildLogTestCase.php
@@ -0,0 +1,117 @@
+<?php
+
+final class HarbormasterBuildLogTestCase
+  extends PhabricatorTestCase {
+
+  public function testBuildLogLineMaps() {
+    $snowman = "\xE2\x98\x83";
+
+    $inputs = array(
+      'no_newlines.log' => array(
+        64,
+        array(
+          str_repeat('AAAAAAAA', 32),
+        ),
+        array(
+          array(64, 0),
+          array(128, 0),
+          array(192, 0),
+          array(255, 0),
+        ),
+      ),
+      'no_newlines_updated.log' => array(
+        64,
+        array_fill(0, 32, 'AAAAAAAA'),
+        array(
+          array(64, 0),
+          array(128, 0),
+          array(192, 0),
+        ),
+      ),
+      'one_newline.log' => array(
+        64,
+        array(
+          str_repeat('AAAAAAAA', 16),
+          "\n",
+          str_repeat('AAAAAAAA', 16),
+        ),
+        array(
+          array(64, 0),
+          array(127, 0),
+          array(191, 1),
+          array(255, 1),
+        ),
+      ),
+      'several_newlines.log' => array(
+        64,
+        array_fill(0, 12, "AAAAAAAAAAAAAAAAAA\n"),
+        array(
+          array(56, 2),
+          array(113, 5),
+          array(170, 8),
+          array(227, 11),
+        ),
+      ),
+      'mixed_newlines.log' => array(
+        64,
+        array(
+          str_repeat('A', 63)."\r",
+          str_repeat('A', 63)."\r\n",
+          str_repeat('A', 63)."\n",
+          str_repeat('A', 63),
+        ),
+        array(
+          array(63, 0),
+          array(127, 1),
+          array(191, 2),
+          array(255, 3),
+        ),
+      ),
+      'more_mixed_newlines.log' => array(
+        64,
+        array(
+          str_repeat('A', 63)."\r",
+          str_repeat('A', 62)."\r\n",
+          str_repeat('A', 63)."\n",
+          str_repeat('A', 63),
+        ),
+        array(
+          array(63, 0),
+          array(128, 2),
+          array(191, 2),
+          array(254, 3),
+        ),
+      ),
+      'emoji.log' => array(
+        64,
+        array(
+          str_repeat($snowman, 64),
+        ),
+        array(
+          array(63, 0),
+          array(126, 0),
+          array(189, 0),
+        ),
+      ),
+    );
+
+    foreach ($inputs as $label => $input) {
+      list($distance, $parts, $expect) = $input;
+
+      $log = id(new HarbormasterBuildLog())
+        ->setByteLength(0);
+
+      foreach ($parts as $part) {
+        $log->updateLineMap($part, $distance);
+      }
+
+      list($actual) = $log->getLineMap();
+
+      $this->assertEqual(
+        $expect,
+        $actual,
+        pht('Line Map for "%s"', $label));
+    }
+  }
+
+}
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
index 6b0b15d8b1..9b983f0733 100644
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
@@ -14,6 +14,7 @@ final class HarbormasterBuildLog
   protected $filePHID;
   protected $byteLength;
   protected $chunkFormat;
+  protected $lineMap = array();
 
   private $buildTarget = self::ATTACHABLE;
   private $rope;
@@ -64,6 +65,9 @@ final class HarbormasterBuildLog
   protected function getConfiguration() {
     return array(
       self::CONFIG_AUX_PHID => true,
+      self::CONFIG_SERIALIZATION => array(
+        'lineMap' => self::SERIALIZATION_JSON,
+      ),
       self::CONFIG_COLUMN_SCHEMA => array(
         // T6203/NULLABILITY
         // It seems like these should be non-nullable? All logs should have a
@@ -369,7 +373,8 @@ final class HarbormasterBuildLog
           $this->writeChunk($encoding_text, $data_size, $append_data);
         }
 
-        $this->byteLength += $data_size;
+        $this->updateLineMap($append_data);
+
         $this->save();
       $this->saveTransaction();
 
@@ -377,6 +382,130 @@ final class HarbormasterBuildLog
     }
   }
 
+  public function updateLineMap($append_data, $marker_distance = null) {
+    $this->byteLength += strlen($append_data);
+
+    if (!$marker_distance) {
+      $marker_distance = (self::CHUNK_BYTE_LIMIT / 2);
+    }
+
+    if (!$this->lineMap) {
+      $this->lineMap = array(
+        array(),
+        0,
+        0,
+        null,
+      );
+    }
+
+    list($map, $map_bytes, $line_count, $prefix) = $this->lineMap;
+
+    $buffer = $append_data;
+
+    if ($prefix) {
+      $prefix = base64_decode($prefix);
+      $buffer = $prefix.$buffer;
+    }
+
+    if ($map) {
+      list($last_marker, $last_count) = last($map);
+    } else {
+      $last_marker = 0;
+      $last_count = 0;
+    }
+
+    $max_utf8_width = 8;
+    $next_marker = $last_marker + $marker_distance;
+
+    $pos = 0;
+    $len = strlen($buffer);
+    while (true) {
+      // If we only have a few bytes left in the buffer, leave it as a prefix
+      // for next time.
+      if (($len - $pos) <= ($max_utf8_width * 2)) {
+        $prefix = substr($buffer, $pos);
+        break;
+      }
+
+      // The next slice we're going to look at is the smaller of:
+      //
+      //   - the number of bytes we need to make it to the next marker; or
+      //   - all the bytes we have left, minus one.
+
+      $slice_length = min(
+        ($marker_distance - $map_bytes),
+        ($len - $pos) - 1);
+
+      // We don't slice all the way to the end for two reasons.
+
+      // First, we want to avoid slicing immediately after a "\r" if we don't
+      // know what the next character is, because we want to make sure to
+      // count "\r\n" as a single newline, rather than counting the "\r" as
+      // a newline and then later counting the "\n" as another newline.
+
+      // Second, we don't want to slice in the middle of a UTF8 character if
+      // we can help it. We may not be able to avoid this, since the whole
+      // buffer may just be binary data, but in most cases we can backtrack
+      // a little bit and try to make it out of emoji or other legitimate
+      // multibyte UTF8 characters which appear in the log.
+
+      $min_width = max(1, $slice_length - $max_utf8_width);
+      while ($slice_length >= $min_width) {
+        $here = $buffer[$pos + ($slice_length - 1)];
+        $next = $buffer[$pos + ($slice_length - 1) + 1];
+
+        // If this is "\r" and the next character is "\n", extend the slice
+        // to include the "\n". Otherwise, we're fine to slice here since we
+        // know we're not in the middle of a UTF8 character.
+        if ($here === "\r") {
+          if ($next === "\n") {
+            $slice_length++;
+          }
+          break;
+        }
+
+        // If the next character is 0x7F or lower, or between 0xC2 and 0xF4,
+        // we're not slicing in the middle of a UTF8 character.
+        $ord = ord($next);
+        if ($ord <= 0x7F || ($ord >= 0xC2 && $ord <= 0xF4)) {
+          break;
+        }
+
+        $slice_length--;
+      }
+
+      $slice = substr($buffer, $pos, $slice_length);
+      $pos += $slice_length;
+
+      $map_bytes += $slice_length;
+      $line_count += count(preg_split("/\r\n|\r|\n/", $slice)) - 1;
+
+      if ($map_bytes >= ($marker_distance - $max_utf8_width)) {
+        $map[] = array(
+          $last_marker + $map_bytes,
+          $last_count + $line_count,
+        );
+
+        $last_count = $last_count + $line_count;
+        $line_count = 0;
+
+        $last_marker = $last_marker + $map_bytes;
+        $map_bytes = 0;
+
+        $next_marker = $last_marker + $marker_distance;
+      }
+    }
+
+    $this->lineMap = array(
+      $map,
+      $map_bytes,
+      $line_count,
+      base64_encode($prefix),
+    );
+
+    return $this;
+  }
+
 
 /* -(  PhabricatorPolicyInterface  )----------------------------------------- */
 
diff --git a/src/applications/harbormaster/worker/HarbormasterLogWorker.php b/src/applications/harbormaster/worker/HarbormasterLogWorker.php
index fc8762de0a..d9d7511c27 100644
--- a/src/applications/harbormaster/worker/HarbormasterLogWorker.php
+++ b/src/applications/harbormaster/worker/HarbormasterLogWorker.php
@@ -57,17 +57,18 @@ final class HarbormasterLogWorker extends HarbormasterWorker {
     $data = $this->getTaskData();
     $is_force = idx($data, 'force');
 
-    if (!$log->getByteLength() || $is_force) {
+    if (!$log->getByteLength() || !$log->getLineMap() || $is_force) {
       $iterator = $log->newDataIterator();
 
-      $byte_length = 0;
+      $log
+        ->setByteLength(0)
+        ->setLineMap(array());
+
       foreach ($iterator as $block) {
-        $byte_length += strlen($block);
+        $log->updateLineMap($block);
       }
 
-      $log
-        ->setByteLength($byte_length)
-        ->save();
+      $log->save();
     }
 
     $format_text = HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT;

From 11d1dc484b867d287a258813510c395407e9151c Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Thu, 22 Feb 2018 19:35:13 -0800
Subject: [PATCH 12/39] Sort of make Harbormaster build logs page properly

Summary: Depends on D19139. Ref T13088. This doesn't actually work, but is close enough that a skilled attacker might be able to briefly deceive a small child.

Test Plan:
  - Viewed some very small logs under very controlled conditions, saw content.
  - Larger logs vaguely do something resembling working correctly.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19141
---
 resources/celerity/map.php                    |   9 +-
 src/__phutil_library_map__.php                |   2 +
 .../PhabricatorHarbormasterApplication.php    |   5 +-
 .../HarbormasterBuildLogRenderController.php  | 562 ++++++++++++++++++
 .../HarbormasterBuildLogViewController.php    |   6 +-
 .../storage/build/HarbormasterBuildLog.php    |  33 +
 .../view/HarbormasterBuildLogView.php         |  30 +-
 .../application/harbormaster/harbormaster.css |  38 ++
 .../harbormaster/behavior-harbormaster-log.js |  43 ++
 9 files changed, 721 insertions(+), 7 deletions(-)
 create mode 100644 src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
 create mode 100644 webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js

diff --git a/resources/celerity/map.php b/resources/celerity/map.php
index 01dd75c901..1361bbd12c 100644
--- a/resources/celerity/map.php
+++ b/resources/celerity/map.php
@@ -78,7 +78,7 @@ return array(
     'rsrc/css/application/feed/feed.css' => 'ecd4ec57',
     'rsrc/css/application/files/global-drag-and-drop.css' => 'b556a948',
     'rsrc/css/application/flag/flag.css' => 'bba8f811',
-    'rsrc/css/application/harbormaster/harbormaster.css' => 'f491c9f4',
+    'rsrc/css/application/harbormaster/harbormaster.css' => 'fecac64f',
     'rsrc/css/application/herald/herald-test.css' => 'a52e323e',
     'rsrc/css/application/herald/herald.css' => 'cd8d0134',
     'rsrc/css/application/maniphest/report.css' => '9b9580b7',
@@ -416,6 +416,7 @@ return array(
     'rsrc/js/application/drydock/drydock-live-operation-status.js' => '901935ef',
     'rsrc/js/application/files/behavior-icon-composer.js' => '8499b6ab',
     'rsrc/js/application/files/behavior-launch-icon-composer.js' => '48086888',
+    'rsrc/js/application/harbormaster/behavior-harbormaster-log.js' => '0844f3c1',
     'rsrc/js/application/herald/HeraldRuleEditor.js' => 'dca75c0e',
     'rsrc/js/application/herald/PathTypeahead.js' => 'f7fc67ec',
     'rsrc/js/application/herald/herald-rule-editor.js' => '7ebaeed3',
@@ -578,7 +579,7 @@ return array(
     'font-fontawesome' => 'e838e088',
     'font-lato' => 'c7ccd872',
     'global-drag-and-drop-css' => 'b556a948',
-    'harbormaster-css' => 'f491c9f4',
+    'harbormaster-css' => 'fecac64f',
     'herald-css' => 'cd8d0134',
     'herald-rule-editor' => 'dca75c0e',
     'herald-test-css' => 'a52e323e',
@@ -635,6 +636,7 @@ return array(
     'javelin-behavior-event-all-day' => 'b41537c9',
     'javelin-behavior-fancy-datepicker' => 'ecf4e799',
     'javelin-behavior-global-drag-and-drop' => '960f6a39',
+    'javelin-behavior-harbormaster-log' => '0844f3c1',
     'javelin-behavior-herald-rule-editor' => '7ebaeed3',
     'javelin-behavior-high-security-warning' => 'a464fe03',
     'javelin-behavior-history-install' => '7ee2b591',
@@ -960,6 +962,9 @@ return array(
       'javelin-stratcom',
       'javelin-workflow',
     ),
+    '0844f3c1' => array(
+      'javelin-behavior',
+    ),
     '08f4ccc3' => array(
       'phui-oi-list-view-css',
     ),
diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
index 01407ed67b..3a7ba1dbbc 100644
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -1230,6 +1230,7 @@ phutil_register_library_map(array(
     'HarbormasterBuildLogDownloadController' => 'applications/harbormaster/controller/HarbormasterBuildLogDownloadController.php',
     'HarbormasterBuildLogPHIDType' => 'applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php',
     'HarbormasterBuildLogQuery' => 'applications/harbormaster/query/HarbormasterBuildLogQuery.php',
+    'HarbormasterBuildLogRenderController' => 'applications/harbormaster/controller/HarbormasterBuildLogRenderController.php',
     'HarbormasterBuildLogTestCase' => 'applications/harbormaster/__tests__/HarbormasterBuildLogTestCase.php',
     'HarbormasterBuildLogView' => 'applications/harbormaster/view/HarbormasterBuildLogView.php',
     'HarbormasterBuildLogViewController' => 'applications/harbormaster/controller/HarbormasterBuildLogViewController.php',
@@ -6519,6 +6520,7 @@ phutil_register_library_map(array(
     'HarbormasterBuildLogDownloadController' => 'HarbormasterController',
     'HarbormasterBuildLogPHIDType' => 'PhabricatorPHIDType',
     'HarbormasterBuildLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
+    'HarbormasterBuildLogRenderController' => 'HarbormasterController',
     'HarbormasterBuildLogTestCase' => 'PhabricatorTestCase',
     'HarbormasterBuildLogView' => 'AphrontView',
     'HarbormasterBuildLogViewController' => 'HarbormasterController',
diff --git a/src/applications/harbormaster/application/PhabricatorHarbormasterApplication.php b/src/applications/harbormaster/application/PhabricatorHarbormasterApplication.php
index b4f62d8e22..103e6bce69 100644
--- a/src/applications/harbormaster/application/PhabricatorHarbormasterApplication.php
+++ b/src/applications/harbormaster/application/PhabricatorHarbormasterApplication.php
@@ -97,7 +97,10 @@ final class PhabricatorHarbormasterApplication extends PhabricatorApplication {
           'buildkite/' => 'HarbormasterBuildkiteHookController',
         ),
         'log/' => array(
-          'view/(?P<id>\d+)/' => 'HarbormasterBuildLogViewController',
+          'view/(?P<id>\d+)/(?:\$(?P<lines>\d+(?:-\d+)?))?'
+            => 'HarbormasterBuildLogViewController',
+          'render/(?P<id>\d+)/(?:\$(?P<lines>\d+(?:-\d+)?))?'
+            => 'HarbormasterBuildLogRenderController',
           'download/(?P<id>\d+)/' => 'HarbormasterBuildLogDownloadController',
         ),
       ),
diff --git a/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
new file mode 100644
index 0000000000..a2322dc51d
--- /dev/null
+++ b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
@@ -0,0 +1,562 @@
+<?php
+
+final class HarbormasterBuildLogRenderController
+  extends HarbormasterController {
+
+  public function handleRequest(AphrontRequest $request) {
+    $viewer = $this->getViewer();
+
+    $id = $request->getURIData('id');
+
+    $log = id(new HarbormasterBuildLogQuery())
+      ->setViewer($viewer)
+      ->withIDs(array($id))
+      ->executeOne();
+    if (!$log) {
+      return new Aphront404Response();
+    }
+
+    $log_size = $this->getTotalByteLength($log);
+
+    $head_lines = $request->getInt('head');
+    if ($head_lines === null) {
+      $head_lines = 8;
+    }
+    $head_lines = min($head_lines, 100);
+    $head_lines = max($head_lines, 0);
+
+    $tail_lines = $request->getInt('tail');
+    if ($tail_lines === null) {
+      $tail_lines = 16;
+    }
+    $tail_lines = min($tail_lines, 100);
+    $tail_lines = max($tail_lines, 0);
+
+    $head_offset = $request->getInt('headOffset');
+    if ($head_offset === null) {
+      $head_offset = 0;
+    }
+
+    $tail_offset = $request->getInt('tailOffset');
+    if ($tail_offset === null) {
+      $tail_offset = $log_size;
+    }
+
+    // Figure out which ranges we're actually going to read. We'll read either
+    // one range (either just at the head, or just at the tail) or two ranges
+    // (one at the head and one at the tail).
+
+    // This gets a little bit tricky because: the ranges may overlap; we just
+    // want to do one big read if there is only a little bit of text left
+    // between the ranges; we may not know where the tail range ends; and we
+    // can only read forward from line map markers, not from any arbitrary
+    // position in the file.
+
+    $bytes_per_line = 140;
+    $body_lines = 8;
+
+    $views = array();
+    if ($head_lines > 0) {
+      $views[] = array(
+        'offset' => $head_offset,
+        'lines' => $head_lines,
+        'direction' => 1,
+      );
+    }
+
+    if ($tail_lines > 0) {
+      $views[] = array(
+        'offset' => $tail_offset,
+        'lines' => $tail_lines,
+        'direction' => -1,
+      );
+    }
+
+    $reads = $views;
+    foreach ($reads as $key => $read) {
+      $offset = $read['offset'];
+
+      $lines = $read['lines'];
+
+      $read_length = 0;
+      $read_length += ($lines * $bytes_per_line);
+      $read_length += ($body_lines * $bytes_per_line);
+
+      $direction = $read['direction'];
+      if ($direction < 0) {
+        $offset -= $read_length;
+        if ($offset < 0) {
+          $offset = 0;
+          $read_length = $log_size;
+        }
+      }
+
+      $position = $log->getReadPosition($offset);
+      list($position_offset, $position_line) = $position;
+      $read_length += ($offset - $position_offset);
+
+      $reads[$key]['fetchOffset'] = $position_offset;
+      $reads[$key]['fetchLength'] = $read_length;
+      $reads[$key]['fetchLine'] = $position_line;
+    }
+
+    $reads = $this->mergeOverlappingReads($reads);
+
+    foreach ($reads as $key => $read) {
+      $data = $log->loadData($read['fetchOffset'], $read['fetchLength']);
+
+      $offset = $read['fetchOffset'];
+      $line = $read['fetchLine'];
+      $lines = $this->getLines($data);
+      $line_data = array();
+      foreach ($lines as $line_text) {
+        $length = strlen($line_text);
+        $line_data[] = array(
+          'offset' => $offset,
+          'length' => $length,
+          'line' => $line,
+          'data' => $line_text,
+        );
+        $line += 1;
+        $offset += $length;
+      }
+
+      $reads[$key]['data'] = $data;
+      $reads[$key]['lines'] = $line_data;
+    }
+
+    foreach ($views as $view_key => $view) {
+      $anchor_byte = $view['offset'];
+
+      $data_key = null;
+      foreach ($reads as $read_key => $read) {
+        $s = $read['fetchOffset'];
+        $e = $s + $read['fetchLength'];
+
+        if (($s <= $anchor_byte) && ($e >= $anchor_byte)) {
+          $data_key = $read_key;
+          break;
+        }
+      }
+
+      if ($data_key === null) {
+        throw new Exception(
+          pht('Unable to find fetch!'));
+      }
+
+      $anchor_key = null;
+      foreach ($reads[$data_key]['lines'] as $line_key => $line) {
+        $s = $line['offset'];
+        $e = $s + $line['length'];
+        if (($s <= $anchor_byte) && ($e >= $anchor_byte)) {
+          $anchor_key = $line_key;
+          break;
+        }
+      }
+
+      if ($anchor_key === null) {
+        throw new Exception(
+          pht(
+            'Unable to find lines.'));
+      }
+
+      if ($direction > 0) {
+        $slice_offset = $line_key;
+      } else {
+        $slice_offset = max(0, $line_key - ($view['lines'] - 1));
+      }
+      $slice_length = $view['lines'];
+
+      $views[$view_key] += array(
+        'sliceKey' => $data_key,
+        'sliceOffset' => $slice_offset,
+        'sliceLength' => $slice_length,
+      );
+    }
+
+    foreach ($views as $view_key => $view) {
+      $slice_key = $view['sliceKey'];
+      $lines = array_slice(
+        $reads[$slice_key]['lines'],
+        $view['sliceOffset'],
+        $view['sliceLength']);
+
+      $data_offset = null;
+      $data_length = null;
+      foreach ($lines as $line) {
+        if ($data_offset === null) {
+          $data_offset = $line['offset'];
+        }
+        $data_length += $line['length'];
+      }
+
+      // If the view cursor starts in the middle of a line, we're going to
+      // strip part of the line.
+      $direction = $view['direction'];
+      if ($direction > 0) {
+        $view_offset = $view['offset'];
+        $view_length = $data_length;
+        if ($data_offset < $view_offset) {
+          $trim = ($view_offset - $data_offset);
+          $view_length -= $trim;
+        }
+      } else {
+        $view_offset = $data_offset;
+        $view_length = $data_length;
+        if ($data_offset + $data_length > $view['offset']) {
+          $view_length -= (($data_offset + $data_length) - $view['offset']);
+        }
+      }
+
+      $views[$view_key] += array(
+        'viewOffset' => $view_offset,
+        'viewLength' => $view_length,
+      );
+    }
+
+    $views = $this->mergeOverlappingViews($views);
+
+    foreach ($views as $view_key => $view) {
+      $slice_key = $view['sliceKey'];
+      $lines = array_slice(
+        $reads[$slice_key]['lines'],
+        $view['sliceOffset'],
+        $view['sliceLength']);
+
+      $view_offset = $view['viewOffset'];
+      foreach ($lines as $line_key => $line) {
+        $line_offset = $line['offset'];
+
+        if ($line_offset >= $view_offset) {
+          break;
+        }
+
+        $trim = ($view_offset - $line_offset);
+        $line_data = substr($line['data'], $trim);
+        if (!strlen($line_data)) {
+          unset($lines[$line_key]);
+          continue;
+        }
+
+        $lines[$line_key]['data'] = $line_data;
+        $lines[$line_key]['length'] = strlen($line_data);
+        $lines[$line_key]['offset'] += $trim;
+        break;
+      }
+
+      $view_end = $view['viewOffset'] + $view['viewLength'];
+      foreach ($lines as $line_key => $line) {
+        $line_end = $line['offset'] + $line['length'];
+        if ($line_end <= $view_end) {
+          break;
+        }
+
+        $trim = ($line_end - $view_end);
+        $line_data = substr($line['data'], -$trim);
+        if (!strlen($line_data)) {
+          unset($lines[$line_key]);
+          continue;
+        }
+
+        $lines[$line_key]['data'] = $line_data;
+        $lines[$line_key]['length'] = strlen($line_data);
+      }
+
+      $views[$view_key]['viewData'] = $lines;
+    }
+
+    $spacer = null;
+    $render = array();
+    foreach ($views as $view) {
+      if ($spacer) {
+        $spacer['tail'] = $view['viewOffset'];
+        $render[] = $spacer;
+      }
+
+      $render[] = $view;
+
+      $spacer = array(
+        'spacer' => true,
+        'head' => ($view['viewOffset'] + $view['viewLength']),
+      );
+    }
+
+    $uri = $log->getURI();
+    $highlight_range = $request->getURIData('lines');
+
+    $rows = array();
+    foreach ($render as $range) {
+      if (isset($range['spacer'])) {
+        $rows[] = phutil_tag(
+          'tr',
+          array(),
+          array(
+            phutil_tag(
+              'th',
+              array(),
+              null),
+            phutil_tag(
+              'td',
+              array(),
+              array(
+                javelin_tag(
+                  'a',
+                  array(
+                    'sigil' => 'harbormaster-log-expand',
+                    'meta' => array(
+                      'headOffset' => $range['head'],
+                      'tailOffset' => $range['tail'],
+                      'head' => 4,
+                    ),
+                  ),
+                  'Show Up ^^^^'),
+                '... '.($range['tail'] - $range['head']).' bytes ...',
+                javelin_tag(
+                  'a',
+                  array(
+                    'sigil' => 'harbormaster-log-expand',
+                    'meta' => array(
+                      'headOffset' => $range['head'],
+                      'tailOffset' => $range['tail'],
+                      'tail' => 4,
+                    ),
+                  ),
+                  'Show Down VVVV'),
+              )),
+          ));
+        continue;
+      }
+
+      $lines = $range['viewData'];
+      foreach ($lines as $line) {
+        $display_line = ($line['line'] + 1);
+        $display_text = ($line['data']);
+
+        $display_line = phutil_tag(
+          'a',
+          array(
+            'href' => $uri.'$'.$display_line,
+          ),
+          $display_line);
+
+        $line_cell = phutil_tag('th', array(), $display_line);
+        $text_cell = phutil_tag('td', array(), $display_text);
+
+        $rows[] = phutil_tag(
+          'tr',
+          array(),
+          array(
+            $line_cell,
+            $text_cell,
+          ));
+      }
+    }
+
+    $table = phutil_tag(
+      'table',
+      array(
+        'class' => 'harbormaster-log-table PhabricatorMonospaced',
+      ),
+      $rows);
+
+    // When this is a normal AJAX request, return the rendered log fragment
+    // in an AJAX payload.
+    if ($request->isAjax()) {
+      return id(new AphrontAjaxResponse())
+        ->setContent(
+          array(
+            'markup' => hsprintf('%s', $table),
+          ));
+    }
+
+    // If the page is being accessed as a standalone page, present a
+    // readable version of the fragment for debugging.
+
+    require_celerity_resource('harbormaster-css');
+
+    $header = pht('Standalone Log Fragment');
+
+    $render_view = id(new PHUIObjectBoxView())
+      ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
+      ->setHeaderText($header)
+      ->appendChild($table);
+
+    $page_view = id(new PHUITwoColumnView())
+      ->setFooter($render_view);
+
+    $crumbs = $this->buildApplicationCrumbs()
+      ->addTextCrumb(pht('Build Log %d', $log->getID()), $log->getURI())
+      ->addTextCrumb(pht('Fragment'))
+      ->setBorder(true);
+
+    return $this->newPage()
+      ->setTitle(
+        array(
+          pht('Build Log %d', $log->getID()),
+          pht('Standalone Fragment'),
+        ))
+      ->setCrumbs($crumbs)
+      ->appendChild($page_view);
+  }
+
+  private function getTotalByteLength(HarbormasterBuildLog $log) {
+    $total_bytes = $log->getByteLength();
+    if ($total_bytes) {
+      return (int)$total_bytes;
+    }
+
+    // TODO: Remove this after enough time has passed for installs to run
+    // log rebuilds or decide they don't care about older logs.
+
+    // Older logs don't have this data denormalized onto the log record unless
+    // an administrator has run `bin/harbormaster rebuild-log --all` or
+    // similar. Try to figure it out by summing up the size of each chunk.
+
+    // Note that the log may also be legitimately empty and have actual size
+    // zero.
+    $chunk = new HarbormasterBuildLogChunk();
+    $conn = $chunk->establishConnection('r');
+
+    $row = queryfx_one(
+      $conn,
+      'SELECT SUM(size) total FROM %T WHERE logID = %d',
+      $chunk->getTableName(),
+      $log->getID());
+
+    return (int)$row['total'];
+  }
+
+  private function getLines($data) {
+    $parts = preg_split("/(\r\n|\r|\n)/", $data, 0, PREG_SPLIT_DELIM_CAPTURE);
+
+    if (last($parts) === '') {
+      array_pop($parts);
+    }
+
+    $lines = array();
+    for ($ii = 0; $ii < count($parts); $ii += 2) {
+      $line = $parts[$ii];
+      if (isset($parts[$ii + 1])) {
+        $line .= $parts[$ii + 1];
+      }
+      $lines[] = $line;
+    }
+
+    return $lines;
+  }
+
+
+  private function mergeOverlappingReads(array $reads) {
+    // Find planned reads which will overlap and merge them into a single
+    // larger read.
+
+    $uk = array_keys($reads);
+    $vk = array_keys($reads);
+
+    foreach ($uk as $ukey) {
+      foreach ($vk as $vkey) {
+        // Don't merge a range into itself, even though they do technically
+        // overlap.
+        if ($ukey === $vkey) {
+          continue;
+        }
+
+        $uread = idx($reads, $ukey);
+        if ($uread === null) {
+          continue;
+        }
+
+        $vread = idx($reads, $vkey);
+        if ($vread === null) {
+          continue;
+        }
+
+        $us = $uread['fetchOffset'];
+        $ue = $us + $uread['fetchLength'];
+
+        $vs = $vread['fetchOffset'];
+        $ve = $vs + $vread['fetchLength'];
+
+        if (($vs > $ue) || ($ve < $us)) {
+          continue;
+        }
+
+        $min = min($us, $vs);
+        $max = max($ue, $ve);
+
+        $reads[$ukey]['fetchOffset'] = $min;
+        $reads[$ukey]['fetchLength'] = ($max - $min);
+        $reads[$ukey]['fetchLine'] = min(
+          $uread['fetchLine'],
+          $vread['fetchLine']);
+
+        unset($reads[$vkey]);
+      }
+    }
+
+    return $reads;
+  }
+
+  private function mergeOverlappingViews(array $views) {
+    $uk = array_keys($views);
+    $vk = array_keys($views);
+
+    $body_lines = 8;
+    $body_bytes = ($body_lines * 140);
+
+    foreach ($uk as $ukey) {
+      foreach ($vk as $vkey) {
+        if ($ukey === $vkey) {
+          continue;
+        }
+
+        $uview = idx($views, $ukey);
+        if ($uview === null) {
+          continue;
+        }
+
+        $vview = idx($views, $vkey);
+        if ($vview === null) {
+          continue;
+        }
+
+        // If these views don't use the same line data, don't try to
+        // merge them.
+        if ($uview['sliceKey'] != $vview['sliceKey']) {
+          continue;
+        }
+
+        // If these views are overlapping or separated by only a few bytes,
+        // merge them into a single view.
+        $us = $uview['viewOffset'];
+        $ue = $us + $uview['viewLength'];
+
+        $vs = $vview['viewOffset'];
+        $ve = $vs + $vview['viewLength'];
+
+        $uss = $uview['sliceOffset'];
+        $use = $uss + $uview['sliceLength'];
+
+        $vss = $vview['sliceOffset'];
+        $vse = $vss + $vview['sliceLength'];
+
+        if ($ue <= $vs) {
+          if (($ue + $body_bytes) >= $vs) {
+            if (($use + $body_lines) >= $vss) {
+              $views[$ukey] = array(
+                'sliceLength' => ($vse - $uss),
+                'viewLength' => ($ve - $us),
+              ) + $views[$ukey];
+
+              unset($views[$vkey]);
+              continue;
+            }
+          }
+        }
+      }
+    }
+
+    return $views;
+  }
+
+}
diff --git a/src/applications/harbormaster/controller/HarbormasterBuildLogViewController.php b/src/applications/harbormaster/controller/HarbormasterBuildLogViewController.php
index 084c48d1f1..41f2aef8e5 100644
--- a/src/applications/harbormaster/controller/HarbormasterBuildLogViewController.php
+++ b/src/applications/harbormaster/controller/HarbormasterBuildLogViewController.php
@@ -4,8 +4,7 @@ final class HarbormasterBuildLogViewController
   extends HarbormasterController {
 
   public function handleRequest(AphrontRequest $request) {
-    $request = $this->getRequest();
-    $viewer = $request->getUser();
+    $viewer = $this->getViewer();
 
     $id = $request->getURIData('id');
 
@@ -21,7 +20,8 @@ final class HarbormasterBuildLogViewController
 
     $log_view = id(new HarbormasterBuildLogView())
       ->setViewer($viewer)
-      ->setBuildLog($log);
+      ->setBuildLog($log)
+      ->setHighlightedLineRange($request->getURIData('lines'));
 
     $crumbs = $this->buildApplicationCrumbs()
       ->addTextCrumb(pht('Build Logs'))
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
index 9b983f0733..64934bbea8 100644
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
@@ -129,6 +129,30 @@ final class HarbormasterBuildLog
       $this->getID());
   }
 
+  public function loadData($offset, $length) {
+    return substr($this->getLogText(), $offset, $length);
+  }
+
+  public function getReadPosition($read_offset) {
+    $position = array(0, 0);
+
+    $map = $this->getLineMap();
+    if (!$map) {
+      throw new Exception(pht('No line map.'));
+    }
+
+    list($map) = $map;
+    foreach ($map as $marker) {
+      list($offset, $count) = $marker;
+      if ($offset > $read_offset) {
+        break;
+      }
+      $position = $marker;
+    }
+
+    return $position;
+  }
+
   public function getLogText() {
     // TODO: Remove this method since it won't scale for big logs.
 
@@ -148,6 +172,15 @@ final class HarbormasterBuildLog
     return "/harbormaster/log/view/{$id}/";
   }
 
+  public function getRenderURI($lines) {
+    if (strlen($lines)) {
+      $lines = '$'.$lines;
+    }
+
+    $id = $this->getID();
+    return "/harbormaster/log/render/{$id}/{$lines}";
+  }
+
 
 /* -(  Chunks  )------------------------------------------------------------- */
 
diff --git a/src/applications/harbormaster/view/HarbormasterBuildLogView.php b/src/applications/harbormaster/view/HarbormasterBuildLogView.php
index 7543524285..81705f3f49 100644
--- a/src/applications/harbormaster/view/HarbormasterBuildLogView.php
+++ b/src/applications/harbormaster/view/HarbormasterBuildLogView.php
@@ -3,6 +3,7 @@
 final class HarbormasterBuildLogView extends AphrontView {
 
   private $log;
+  private $highlightedLineRange;
 
   public function setBuildLog(HarbormasterBuildLog $log) {
     $this->log = $log;
@@ -13,6 +14,15 @@ final class HarbormasterBuildLogView extends AphrontView {
     return $this->log;
   }
 
+  public function setHighlightedLineRange($range) {
+    $this->highlightedLineRange = $range;
+    return $this;
+  }
+
+  public function getHighlightedLineRange() {
+    return $this->highlightedLineRange;
+  }
+
   public function render() {
     $viewer = $this->getViewer();
     $log = $this->getBuildLog();
@@ -34,10 +44,28 @@ final class HarbormasterBuildLogView extends AphrontView {
 
     $header->addActionLink($download_button);
 
+    $content_id = celerity_generate_unique_node_id();
+    $content_div = javelin_tag(
+      'div',
+      array(
+        'id' => $content_id,
+        'class' => 'harbormaster-log-view-loading',
+      ),
+      pht('Loading...'));
+
+    require_celerity_resource('harbormaster-css');
+
+    Javelin::initBehavior(
+      'harbormaster-log',
+      array(
+        'contentNodeID' => $content_id,
+        'renderURI' => $log->getRenderURI($this->getHighlightedLineRange()),
+      ));
+
     $box_view = id(new PHUIObjectBoxView())
       ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
       ->setHeader($header)
-      ->appendChild('...');
+      ->appendChild($content_div);
 
     return $box_view;
   }
diff --git a/webroot/rsrc/css/application/harbormaster/harbormaster.css b/webroot/rsrc/css/application/harbormaster/harbormaster.css
index 4bfd2d6cbe..f55139ade0 100644
--- a/webroot/rsrc/css/application/harbormaster/harbormaster.css
+++ b/webroot/rsrc/css/application/harbormaster/harbormaster.css
@@ -30,3 +30,41 @@
   text-overflow: ellipsis;
   color: {$lightgreytext};
 }
+
+.harbormaster-log-view-loading {
+  padding: 8px;
+  text-align: center;
+  color: {$lightgreytext};
+}
+
+.harbormaster-log-table th {
+  background-color: {$paste.highlight};
+  border-right: 1px solid {$paste.border};
+
+  -moz-user-select: -moz-none;
+  -khtml-user-select: none;
+  -webkit-user-select: none;
+  -ms-user-select: none;
+  user-select: none;
+}
+
+.harbormaster-log-table th a {
+  display: block;
+  color: {$darkbluetext};
+  text-align: right;
+  padding: 2px 6px 1px 12px;
+}
+
+.harbormaster-log-table th a:hover {
+  background: {$paste.border};
+}
+
+.harbormaster-log-table td {
+  white-space: pre-wrap;
+  padding: 2px 8px 1px;
+  width: 100%;
+}
+
+.harbormaster-log-table tr.harbormaster-log-highlighted td {
+  background: {$paste.highlight};
+}
diff --git a/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js b/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js
new file mode 100644
index 0000000000..d011233afd
--- /dev/null
+++ b/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js
@@ -0,0 +1,43 @@
+/**
+ * @provides javelin-behavior-harbormaster-log
+ * @requires javelin-behavior
+ */
+
+JX.behavior('harbormaster-log', function(config) {
+  var contentNode = JX.$(config.contentNodeID);
+
+  JX.DOM.listen(contentNode, 'click', 'harbormaster-log-expand', function(e) {
+    if (!e.isNormalClick()) {
+      return;
+    }
+
+    e.kill();
+
+    var row = e.getNode('tag:tr');
+    var data = e.getNodeData('harbormaster-log-expand');
+
+    var uri = new JX.URI(config.renderURI)
+      .addQueryParams(data);
+
+    var request = new JX.Request(uri, function(r) {
+      var result = JX.$H(r.markup).getNode();
+      var rows = JX.DOM.scry(result, 'tr');
+
+      JX.DOM.replace(row, rows);
+    });
+
+    request.send();
+  });
+
+  function onresponse(r) {
+    JX.DOM.alterClass(contentNode, 'harbormaster-log-view-loading', false);
+
+    JX.DOM.setContent(contentNode, JX.$H(r.markup));
+  }
+
+  var uri = new JX.URI(config.renderURI);
+
+  new JX.Request(uri, onresponse)
+    .send();
+
+});

From f450c6c55bfdc3680a1895842ac17765a60e3d15 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Mon, 26 Feb 2018 17:46:49 -0800
Subject: [PATCH 13/39] Fix some of the most egregious errors in Harbormaster
 log paging

Summary:
Depends on D19141. Ref T13088. Some of the fundamental log behaviors like "loading the correct rows" are now a bit better behaved.

The UI is a little less garbage, too.

Test Plan: Viewed some logs and loaded more context by clicking the buttons.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19142
---
 resources/celerity/map.php                    |  14 +-
 .../HarbormasterBuildLogRenderController.php  | 196 ++++++++++++++----
 .../application/harbormaster/harbormaster.css |  64 +++++-
 .../harbormaster/behavior-harbormaster-log.js |   4 +-
 4 files changed, 220 insertions(+), 58 deletions(-)

diff --git a/resources/celerity/map.php b/resources/celerity/map.php
index 1361bbd12c..7907d3e18f 100644
--- a/resources/celerity/map.php
+++ b/resources/celerity/map.php
@@ -78,7 +78,7 @@ return array(
     'rsrc/css/application/feed/feed.css' => 'ecd4ec57',
     'rsrc/css/application/files/global-drag-and-drop.css' => 'b556a948',
     'rsrc/css/application/flag/flag.css' => 'bba8f811',
-    'rsrc/css/application/harbormaster/harbormaster.css' => 'fecac64f',
+    'rsrc/css/application/harbormaster/harbormaster.css' => 'c7e29d9e',
     'rsrc/css/application/herald/herald-test.css' => 'a52e323e',
     'rsrc/css/application/herald/herald.css' => 'cd8d0134',
     'rsrc/css/application/maniphest/report.css' => '9b9580b7',
@@ -416,7 +416,7 @@ return array(
     'rsrc/js/application/drydock/drydock-live-operation-status.js' => '901935ef',
     'rsrc/js/application/files/behavior-icon-composer.js' => '8499b6ab',
     'rsrc/js/application/files/behavior-launch-icon-composer.js' => '48086888',
-    'rsrc/js/application/harbormaster/behavior-harbormaster-log.js' => '0844f3c1',
+    'rsrc/js/application/harbormaster/behavior-harbormaster-log.js' => 'be6974cc',
     'rsrc/js/application/herald/HeraldRuleEditor.js' => 'dca75c0e',
     'rsrc/js/application/herald/PathTypeahead.js' => 'f7fc67ec',
     'rsrc/js/application/herald/herald-rule-editor.js' => '7ebaeed3',
@@ -579,7 +579,7 @@ return array(
     'font-fontawesome' => 'e838e088',
     'font-lato' => 'c7ccd872',
     'global-drag-and-drop-css' => 'b556a948',
-    'harbormaster-css' => 'fecac64f',
+    'harbormaster-css' => 'c7e29d9e',
     'herald-css' => 'cd8d0134',
     'herald-rule-editor' => 'dca75c0e',
     'herald-test-css' => 'a52e323e',
@@ -636,7 +636,7 @@ return array(
     'javelin-behavior-event-all-day' => 'b41537c9',
     'javelin-behavior-fancy-datepicker' => 'ecf4e799',
     'javelin-behavior-global-drag-and-drop' => '960f6a39',
-    'javelin-behavior-harbormaster-log' => '0844f3c1',
+    'javelin-behavior-harbormaster-log' => 'be6974cc',
     'javelin-behavior-herald-rule-editor' => '7ebaeed3',
     'javelin-behavior-high-security-warning' => 'a464fe03',
     'javelin-behavior-history-install' => '7ee2b591',
@@ -962,9 +962,6 @@ return array(
       'javelin-stratcom',
       'javelin-workflow',
     ),
-    '0844f3c1' => array(
-      'javelin-behavior',
-    ),
     '08f4ccc3' => array(
       'phui-oi-list-view-css',
     ),
@@ -1892,6 +1889,9 @@ return array(
       'javelin-util',
       'javelin-request',
     ),
+    'be6974cc' => array(
+      'javelin-behavior',
+    ),
     'bea6e7f4' => array(
       'javelin-install',
       'javelin-dom',
diff --git a/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
index a2322dc51d..c1c4275982 100644
--- a/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
+++ b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
@@ -61,6 +61,7 @@ final class HarbormasterBuildLogRenderController
         'offset' => $head_offset,
         'lines' => $head_lines,
         'direction' => 1,
+        'limit' => $tail_offset,
       );
     }
 
@@ -69,6 +70,7 @@ final class HarbormasterBuildLogRenderController
         'offset' => $tail_offset,
         'lines' => $tail_lines,
         'direction' => -1,
+        'limit' => $head_offset,
       );
     }
 
@@ -128,6 +130,10 @@ final class HarbormasterBuildLogRenderController
     foreach ($views as $view_key => $view) {
       $anchor_byte = $view['offset'];
 
+      if ($view['direction'] < 0) {
+        $anchor_byte = $anchor_byte - 1;
+      }
+
       $data_key = null;
       foreach ($reads as $read_key => $read) {
         $s = $read['fetchOffset'];
@@ -148,7 +154,8 @@ final class HarbormasterBuildLogRenderController
       foreach ($reads[$data_key]['lines'] as $line_key => $line) {
         $s = $line['offset'];
         $e = $s + $line['length'];
-        if (($s <= $anchor_byte) && ($e >= $anchor_byte)) {
+
+        if (($s <= $anchor_byte) && ($e > $anchor_byte)) {
           $anchor_key = $line_key;
           break;
         }
@@ -161,9 +168,9 @@ final class HarbormasterBuildLogRenderController
       }
 
       if ($direction > 0) {
-        $slice_offset = $line_key;
+        $slice_offset = $anchor_key;
       } else {
-        $slice_offset = max(0, $line_key - ($view['lines'] - 1));
+        $slice_offset = max(0, $anchor_key - ($view['lines'] - 1));
       }
       $slice_length = $view['lines'];
 
@@ -200,12 +207,23 @@ final class HarbormasterBuildLogRenderController
           $trim = ($view_offset - $data_offset);
           $view_length -= $trim;
         }
+
+        $limit = $view['limit'];
+        if ($limit < ($view_offset + $view_length)) {
+          $view_length = ($limit - $view_offset);
+        }
       } else {
         $view_offset = $data_offset;
         $view_length = $data_length;
         if ($data_offset + $data_length > $view['offset']) {
           $view_length -= (($data_offset + $data_length) - $view['offset']);
         }
+
+        $limit = $view['limit'];
+        if ($limit > $view_offset) {
+          $view_length -= ($limit - $view_offset);
+          $view_offset = $limit;
+        }
       }
 
       $views[$view_key] += array(
@@ -232,12 +250,12 @@ final class HarbormasterBuildLogRenderController
         }
 
         $trim = ($view_offset - $line_offset);
-        $line_data = substr($line['data'], $trim);
-        if (!strlen($line_data)) {
+        if ($trim && ($trim >= strlen($line['data']))) {
           unset($lines[$line_key]);
           continue;
         }
 
+        $line_data = substr($line['data'], $trim);
         $lines[$line_key]['data'] = $line_data;
         $lines[$line_key]['length'] = strlen($line_data);
         $lines[$line_key]['offset'] += $trim;
@@ -248,16 +266,16 @@ final class HarbormasterBuildLogRenderController
       foreach ($lines as $line_key => $line) {
         $line_end = $line['offset'] + $line['length'];
         if ($line_end <= $view_end) {
-          break;
+          continue;
         }
 
         $trim = ($line_end - $view_end);
-        $line_data = substr($line['data'], -$trim);
-        if (!strlen($line_data)) {
+        if ($trim && ($trim >= strlen($line['data']))) {
           unset($lines[$line_key]);
           continue;
         }
 
+        $line_data = substr($line['data'], -$trim);
         $lines[$line_key]['data'] = $line_data;
         $lines[$line_key]['length'] = strlen($line_data);
       }
@@ -267,6 +285,17 @@ final class HarbormasterBuildLogRenderController
 
     $spacer = null;
     $render = array();
+
+    $head_view = head($views);
+    if ($head_view['viewOffset'] > $head_offset) {
+      $render[] = array(
+        'spacer' => true,
+        'head' => $head_offset,
+        'tail' => $head_view['viewOffset'],
+      );
+    }
+
+
     foreach ($views as $view) {
       if ($spacer) {
         $spacer['tail'] = $view['viewOffset'];
@@ -281,49 +310,22 @@ final class HarbormasterBuildLogRenderController
       );
     }
 
+    $tail_view = last($views);
+    if ($tail_view['viewOffset'] + $tail_view['viewLength'] < $tail_offset) {
+      $render[] = array(
+        'spacer' => true,
+        'head' => $tail_view['viewOffset'] + $tail_view['viewLength'],
+        'tail' => $tail_offset,
+      );
+    }
+
     $uri = $log->getURI();
     $highlight_range = $request->getURIData('lines');
 
     $rows = array();
     foreach ($render as $range) {
       if (isset($range['spacer'])) {
-        $rows[] = phutil_tag(
-          'tr',
-          array(),
-          array(
-            phutil_tag(
-              'th',
-              array(),
-              null),
-            phutil_tag(
-              'td',
-              array(),
-              array(
-                javelin_tag(
-                  'a',
-                  array(
-                    'sigil' => 'harbormaster-log-expand',
-                    'meta' => array(
-                      'headOffset' => $range['head'],
-                      'tailOffset' => $range['tail'],
-                      'head' => 4,
-                    ),
-                  ),
-                  'Show Up ^^^^'),
-                '... '.($range['tail'] - $range['head']).' bytes ...',
-                javelin_tag(
-                  'a',
-                  array(
-                    'sigil' => 'harbormaster-log-expand',
-                    'meta' => array(
-                      'headOffset' => $range['head'],
-                      'tailOffset' => $range['tail'],
-                      'tail' => 4,
-                    ),
-                  ),
-                  'Show Down VVVV'),
-              )),
-          ));
+        $rows[] = $this->renderExpandRow($range);
         continue;
       }
 
@@ -559,4 +561,108 @@ final class HarbormasterBuildLogRenderController
     return $views;
   }
 
+  private function renderExpandRow($range) {
+
+    $icon_up = id(new PHUIIconView())
+      ->setIcon('fa-chevron-up');
+
+    $icon_down = id(new PHUIIconView())
+      ->setIcon('fa-chevron-down');
+
+    $up_text = array(
+      pht('Show More Above'),
+      ' ',
+      $icon_up,
+    );
+
+    $expand_up = javelin_tag(
+      'a',
+      array(
+        'sigil' => 'harbormaster-log-expand',
+        'meta' => array(
+          'headOffset' => $range['head'],
+          'tailOffset' => $range['tail'],
+          'head' => 4,
+          'tail' => 0,
+        ),
+      ),
+      $up_text);
+
+    $mid_text = pht(
+      'Show More (%s bytes Hidden)',
+      new PhutilNumber($range['tail'] - $range['head']));
+
+    $expand_mid = javelin_tag(
+      'a',
+      array(
+        'sigil' => 'harbormaster-log-expand',
+        'meta' => array(
+          'headOffset' => $range['head'],
+          'tailOffset' => $range['tail'],
+          'head' => 2,
+          'tail' => 2,
+        ),
+      ),
+      $mid_text);
+
+    $down_text = array(
+      $icon_down,
+      ' ',
+      pht('Show More Below'),
+    );
+
+    $expand_down = javelin_tag(
+      'a',
+      array(
+        'sigil' => 'harbormaster-log-expand',
+        'meta' => array(
+          'headOffset' => $range['head'],
+          'tailOffset' => $range['tail'],
+          'head' => 0,
+          'tail' => 4,
+        ),
+      ),
+      $down_text);
+
+    $expand_cells = array(
+      phutil_tag(
+        'td',
+        array(
+          'class' => 'harbormaster-log-expand-up',
+        ),
+        $expand_up),
+      phutil_tag(
+        'td',
+        array(
+          'class' => 'harbormaster-log-expand-mid',
+        ),
+        $expand_mid),
+      phutil_tag(
+        'td',
+        array(
+          'class' => 'harbormaster-log-expand-down',
+        ),
+        $expand_down),
+    );
+    $expand_row = phutil_tag('tr', array(), $expand_cells);
+    $expand_table = phutil_tag(
+      'table',
+      array(
+        'class' => 'harbormaster-log-expand-table',
+      ),
+      $expand_row);
+
+    $cells = array(
+      phutil_tag('th', array()),
+      phutil_tag(
+        'td',
+        array(
+          'class' => 'harbormaster-log-expand-cell',
+        ),
+        $expand_table),
+    );
+
+    return phutil_tag('tr', array(), $cells);
+  }
+
 }
diff --git a/webroot/rsrc/css/application/harbormaster/harbormaster.css b/webroot/rsrc/css/application/harbormaster/harbormaster.css
index f55139ade0..1c389a4383 100644
--- a/webroot/rsrc/css/application/harbormaster/harbormaster.css
+++ b/webroot/rsrc/css/application/harbormaster/harbormaster.css
@@ -37,7 +37,7 @@
   color: {$lightgreytext};
 }
 
-.harbormaster-log-table th {
+.harbormaster-log-table > tbody > tr > th {
   background-color: {$paste.highlight};
   border-right: 1px solid {$paste.border};
 
@@ -48,23 +48,77 @@
   user-select: none;
 }
 
-.harbormaster-log-table th a {
+.harbormaster-log-table > tbody > tr > th a {
   display: block;
   color: {$darkbluetext};
   text-align: right;
   padding: 2px 6px 1px 12px;
 }
 
-.harbormaster-log-table th a:hover {
+.harbormaster-log-table > tbody > tr > th a:hover {
   background: {$paste.border};
 }
 
-.harbormaster-log-table td {
+.harbormaster-log-table > tbody > tr > td {
   white-space: pre-wrap;
   padding: 2px 8px 1px;
   width: 100%;
 }
 
-.harbormaster-log-table tr.harbormaster-log-highlighted td {
+.harbormaster-log-table > tbody > tr > td.harbormaster-log-expand-cell {
+  padding: 4px 0;
+}
+
+.harbormaster-log-table tr.harbormaster-log-highlighted > td {
   background: {$paste.highlight};
 }
+
+.harbormaster-log-expand-table {
+  width: 100%;
+  background: {$paste.highlight};
+  border-top: 1px solid {$paste.border};
+  border-bottom: 1px solid {$paste.border};
+}
+
+.harbormaster-log-expand-table a {
+  display: block;
+  padding: 6px 16px;
+  color: {$darkbluetext};
+}
+
+.device-desktop .harbormaster-log-expand-table a:hover {
+  background: {$paste.border};
+  text-decoration: none;
+}
+
+.harbormaster-log-expand-table td {
+  vertical-align: middle;
+  font: 13px 'Segoe UI', 'Segoe UI Emoji', 'Segoe UI Symbol', 'Lato',
+    'Helvetica Neue', Helvetica, Arial, sans-serif;
+}
+
+
+.harbormaster-log-expand-up {
+  text-align: right;
+  width: 50%;
+}
+
+.harbormaster-log-expand-up .phui-icon-view {
+  margin: 0 0 0px 4px;
+}
+
+.harbormaster-log-expand-mid {
+  text-align: center;
+  white-space: nowrap;
+  border-left: 1px solid {$paste.border};
+  border-right: 1px solid {$paste.border};
+}
+
+.harbormaster-log-expand-down {
+  text-align: left;
+  width: 50%;
+}
+
+.harbormaster-log-expand-down .phui-icon-view {
+  margin: 0 4px 0 0;
+}
diff --git a/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js b/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js
index d011233afd..38baca6199 100644
--- a/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js
+++ b/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js
@@ -14,6 +14,8 @@ JX.behavior('harbormaster-log', function(config) {
     e.kill();
 
     var row = e.getNode('tag:tr');
+    row = JX.DOM.findAbove(row, 'tr');
+
     var data = e.getNodeData('harbormaster-log-expand');
 
     var uri = new JX.URI(config.renderURI)
@@ -21,7 +23,7 @@ JX.behavior('harbormaster-log', function(config) {
 
     var request = new JX.Request(uri, function(r) {
       var result = JX.$H(r.markup).getNode();
-      var rows = JX.DOM.scry(result, 'tr');
+      var rows = [].slice.apply(result.firstChild.childNodes);
 
       JX.DOM.replace(row, rows);
     });

From dba4c4bdf6a850ab9f693ecebedbd0d90aad08d4 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Tue, 27 Feb 2018 06:56:15 -0800
Subject: [PATCH 14/39] Emit a "Content-Security-Policy" HTTP header

Summary:
See PHI399. Ref T4340. This header provides an additional layer of protection against various attacks, including XSS attacks which embed inline `<script ...>` or `onhover="..."` content into the document.

**style-src**: The "unsafe-inline" directive affects both `style="..."` and `<style>`. We use a lot of `style="..."`, some very legitimately, so we can't realistically get away from this any time soon. We only use one `<style>` (for monospaced font preferences) but can't disable `<style>` without disabling `style="..."`.

**img-src**: We use "data:" URIs to inline small images into CSS, and there's a significant performance benefit from doing this. There doesn't seem to be a way to allow "data" URIs in CSS without allowing them in the document itself.

**script-src** and **frame-src**: For a small number of flows (Recaptcha, Stripe) we embed external javascript, some of which embeds child elements (or additional resources) into the document. We now whitelist these narrowly on the respective pages.

This won't work with Quicksand, so I've blacklisted it for now.

**connect-src**: We need to include `'self'` for AJAX to work, and any websocket URIs.

**Clickjacking**: We now have three layers of protection:

  - X-Frame-Options: works in older browsers.
  - `frame-ancestors 'none'`: does the same thing.
  - Explicit framebust in JX.Stratcom after initialization: works in ancient IE.

We could probably drop the explicit framebust but it wasn't difficult to retain.

**script tags**: We previously used an inline `<script>` tag to start Javelin. I've moved this to `<data data-javelin-init ...>` tags, which seems to work properly.

**`__DEV__`**: We previously used an inline `<script>` tag to set the `__DEV__` mode flag. I tried using the "initialization" tags for this, but they fire too late. I moved it to `<html data-developer-mode="1">`, which seems OK everywhere.

**CSP Scope**: Only the CSP header on the original request appears to matter -- you can't refine the scope by emitting headers on CSS/JS. To reduce confusion, I disabled the headers on those response types. More headers could be disabled, although we're likely already deep in the land of diminishing returns.

**Initialization**: The initialization sequence has changed slightly. Previously, we waited for the <script> in bottom of the document to evaluate. Now, we go fishing for tags when domcontentready fires.

Test Plan:
  - Browsed around in Firefox, Safari and Chrome looking for console warnings. Interacted with various Javascript behaviors. Enabled Quicksand.
  - Disabled all the framebusting, launched a clickjacking attack, verified that each layer of protection is individually effective.
  - Verified that the XHProf iframe in Darkconsole and the PHPAST frame layout work properly.
  - Enabled notifications, verified no complaints about connecting to Aphlict.
  - Hit `__DEV__` mode warnings based on the new data attribute.
  - Tried to do sketchy stuff with `data:` URIs and SVGs. This works but doesn't seem to be able to do anything dangerous.
  - Went through the Stripe and Recaptcha workflows.
  - Dumped and examined the CSP headers with `curl`, etc.
  - Added a raw <script> tag to a page (as though I'd found an XSS attack), verified it was no longer executed.

Maniphest Tasks: T4340

Differential Revision: https://secure.phabricator.com/D19143
---
 resources/celerity/map.php                    |  22 +--
 src/aphront/response/AphrontResponse.php      | 127 +++++++++++++++++-
 .../CelerityStaticResourceResponse.php        |  83 ++++++++----
 .../controller/CelerityResourceController.php |   5 +
 .../PhortuneStripePaymentProvider.php         |   8 +-
 .../control/AphrontFormRecaptchaControl.php   |  26 ++--
 src/view/page/AphrontPageView.php             |   9 +-
 src/view/page/PhabricatorBarePageView.php     |  10 +-
 src/view/page/PhabricatorStandardPageView.php |  28 +++-
 .../rsrc/externals/javelin/core/Stratcom.js   |  31 ++++-
 webroot/rsrc/externals/javelin/core/init.js   |  43 ++++--
 11 files changed, 322 insertions(+), 70 deletions(-)

diff --git a/resources/celerity/map.php b/resources/celerity/map.php
index 7907d3e18f..1f3884f8da 100644
--- a/resources/celerity/map.php
+++ b/resources/celerity/map.php
@@ -10,7 +10,7 @@ return array(
     'conpherence.pkg.css' => 'e68cf1fa',
     'conpherence.pkg.js' => '15191c65',
     'core.pkg.css' => '2fa91e14',
-    'core.pkg.js' => 'dc13d4b7',
+    'core.pkg.js' => '7aa5bd92',
     'darkconsole.pkg.js' => '1f9a31bc',
     'differential.pkg.css' => '113e692c',
     'differential.pkg.js' => 'f6d809c0',
@@ -211,12 +211,12 @@ return array(
     'rsrc/externals/font/lato/lato-regular.woff' => '13d39fe2',
     'rsrc/externals/font/lato/lato-regular.woff2' => '57a9f742',
     'rsrc/externals/javelin/core/Event.js' => '2ee659ce',
-    'rsrc/externals/javelin/core/Stratcom.js' => '6ad39b6f',
+    'rsrc/externals/javelin/core/Stratcom.js' => '327f418a',
     'rsrc/externals/javelin/core/__tests__/event-stop-and-kill.js' => '717554e4',
     'rsrc/externals/javelin/core/__tests__/install.js' => 'c432ee85',
     'rsrc/externals/javelin/core/__tests__/stratcom.js' => '88bf7313',
     'rsrc/externals/javelin/core/__tests__/util.js' => 'e251703d',
-    'rsrc/externals/javelin/core/init.js' => '3010e992',
+    'rsrc/externals/javelin/core/init.js' => '638a4e2b',
     'rsrc/externals/javelin/core/init_node.js' => 'c234aded',
     'rsrc/externals/javelin/core/install.js' => '05270951',
     'rsrc/externals/javelin/core/util.js' => '93cc50d6',
@@ -722,7 +722,7 @@ return array(
     'javelin-install' => '05270951',
     'javelin-json' => '69adf288',
     'javelin-leader' => '7f243deb',
-    'javelin-magical-init' => '3010e992',
+    'javelin-magical-init' => '638a4e2b',
     'javelin-mask' => '8a41885b',
     'javelin-quicksand' => '6b8ef10b',
     'javelin-reactor' => '2b8de964',
@@ -735,7 +735,7 @@ return array(
     'javelin-router' => '29274e2b',
     'javelin-scrollbar' => '9065f639',
     'javelin-sound' => '949c0fe5',
-    'javelin-stratcom' => '6ad39b6f',
+    'javelin-stratcom' => '327f418a',
     'javelin-tokenizer' => '8d3bc1b2',
     'javelin-typeahead' => '70baed2f',
     'javelin-typeahead-composite-source' => '503e17fd',
@@ -1131,6 +1131,12 @@ return array(
       'javelin-dom',
       'javelin-workflow',
     ),
+    '327f418a' => array(
+      'javelin-install',
+      'javelin-event',
+      'javelin-util',
+      'javelin-magical-init',
+    ),
     '358b8c04' => array(
       'javelin-install',
       'javelin-util',
@@ -1446,12 +1452,6 @@ return array(
     '69adf288' => array(
       'javelin-install',
     ),
-    '6ad39b6f' => array(
-      'javelin-install',
-      'javelin-event',
-      'javelin-util',
-      'javelin-magical-init',
-    ),
     '6b8ef10b' => array(
       'javelin-install',
     ),
diff --git a/src/aphront/response/AphrontResponse.php b/src/aphront/response/AphrontResponse.php
index 6c52e417e3..47ec2e465b 100644
--- a/src/aphront/response/AphrontResponse.php
+++ b/src/aphront/response/AphrontResponse.php
@@ -7,9 +7,11 @@ abstract class AphrontResponse extends Phobject {
   private $canCDN;
   private $responseCode = 200;
   private $lastModified = null;
-
+  private $contentSecurityPolicyURIs;
+  private $disableContentSecurityPolicy;
   protected $frameable;
 
+
   public function setRequest($request) {
     $this->request = $request;
     return $this;
@@ -19,6 +21,32 @@ abstract class AphrontResponse extends Phobject {
     return $this->request;
   }
 
+  final public function addContentSecurityPolicyURI($kind, $uri) {
+    if ($this->contentSecurityPolicyURIs === null) {
+      $this->contentSecurityPolicyURIs = array(
+         'script' => array(),
+         'connect' => array(),
+         'frame' => array(),
+       );
+    }
+
+    if (!isset($this->contentSecurityPolicyURIs[$kind])) {
+      throw new Exception(
+        pht(
+          'Unknown Content-Security-Policy URI kind "%s".',
+          $kind));
+    }
+
+    $this->contentSecurityPolicyURIs[$kind][] = (string)$uri;
+
+    return $this;
+  }
+
+  final public function setDisableContentSecurityPolicy($disable) {
+    $this->disableContentSecurityPolicy = $disable;
+    return $this;
+  }
+
 
 /* -(  Content  )------------------------------------------------------------ */
 
@@ -59,9 +87,106 @@ abstract class AphrontResponse extends Phobject {
       );
     }
 
+    $csp = $this->newContentSecurityPolicyHeader();
+    if ($csp !== null) {
+      $headers[] = array('Content-Security-Policy', $csp);
+    }
+
     return $headers;
   }
 
+  private function newContentSecurityPolicyHeader() {
+    if ($this->disableContentSecurityPolicy) {
+      return null;
+    }
+
+    $csp = array();
+
+    $cdn = PhabricatorEnv::getEnvConfig('security.alternate-file-domain');
+    if ($cdn) {
+      $default = $this->newContentSecurityPolicySource($cdn);
+    } else {
+      $default = "'self'";
+    }
+
+    $csp[] = "default-src {$default}";
+
+    // We use "data:" URIs to inline small images into CSS. This policy allows
+    // "data:" URIs to be used anywhere, but there doesn't appear to be a way
+    // to say that "data:" URIs are okay in CSS files but not in the document.
+    $csp[] = "img-src {$default} data:";
+
+    // We use inline style="..." attributes in various places, many of which
+    // are legitimate. We also currently use a <style> tag to implement the
+    // "Monospaced Font Preference" setting.
+    $csp[] = "style-src {$default} 'unsafe-inline'";
+
+    // On a small number of pages, including the Stripe workflow and the
+    // ReCAPTCHA challenge, we embed external Javascript directly.
+    $csp[] = $this->newContentSecurityPolicy('script', $default);
+
+    // We need to specify that we can connect to ourself in order for AJAX
+    // requests to work.
+    $csp[] = $this->newContentSecurityPolicy('connect', "'self'");
+
+    // DarkConsole and PHPAST both use frames to render some content.
+    $csp[] = $this->newContentSecurityPolicy('frame', "'self'");
+
+    // This is a more modern flavor of of "X-Frame-Options" and prevents
+    // clickjacking attacks where the page is included in a tiny iframe and
+    // the user is convinced to click a element on the page, which really
+    // clicks a dangerous button hidden under a picture of a cat.
+    if ($this->frameable) {
+      $csp[] = "frame-ancestors 'self'";
+    } else {
+      $csp[] = "frame-ancestors 'none'";
+    }
+
+    $csp = implode('; ', $csp);
+
+    return $csp;
+  }
+
+  private function newContentSecurityPolicy($type, $defaults) {
+    if ($defaults === null) {
+      $sources = array();
+    } else {
+      $sources = (array)$defaults;
+    }
+
+    $uris = $this->contentSecurityPolicyURIs;
+    if (isset($uris[$type])) {
+      foreach ($uris[$type] as $uri) {
+        $sources[] = $this->newContentSecurityPolicySource($uri);
+      }
+    }
+    $sources = array_unique($sources);
+
+    return "{$type}-src ".implode(' ', $sources);
+  }
+
+  private function newContentSecurityPolicySource($uri) {
+    // Some CSP URIs are ultimately user controlled (like notification server
+    // URIs and CDN URIs) so attempt to stop an attacker from injecting an
+    // unsafe source (like 'unsafe-eval') into the CSP header.
+
+    $uri = id(new PhutilURI($uri))
+      ->setPath(null)
+      ->setFragment(null)
+      ->setQueryParams(array());
+
+    $uri = (string)$uri;
+    if (preg_match('/[ ;\']/', $uri)) {
+      throw new Exception(
+        pht(
+          'Attempting to emit a response with an unsafe source ("%s") in the '.
+          'Content-Security-Policy header.',
+          $uri));
+    }
+
+    return $uri;
+  }
+
   public function setCacheDurationInSeconds($duration) {
     $this->cacheable = $duration;
     return $this;
diff --git a/src/applications/celerity/CelerityStaticResourceResponse.php b/src/applications/celerity/CelerityStaticResourceResponse.php
index 7db6c2741b..783dd96570 100644
--- a/src/applications/celerity/CelerityStaticResourceResponse.php
+++ b/src/applications/celerity/CelerityStaticResourceResponse.php
@@ -17,6 +17,7 @@ final class CelerityStaticResourceResponse extends Phobject {
   private $behaviors = array();
   private $hasRendered = array();
   private $postprocessorKey;
+  private $contentSecurityPolicyURIs = array();
 
   public function __construct() {
     if (isset($_REQUEST['__metablock__'])) {
@@ -37,6 +38,15 @@ final class CelerityStaticResourceResponse extends Phobject {
     return $this->metadataBlock.'_'.$id;
   }
 
+  public function addContentSecurityPolicyURI($kind, $uri) {
+    $this->contentSecurityPolicyURIs[$kind][] = $uri;
+    return $this;
+  }
+
+  public function getContentSecurityPolicyURIMap() {
+    return $this->contentSecurityPolicyURIs;
+  }
+
   public function getMetadataBlock() {
     return $this->metadataBlock;
   }
@@ -196,23 +206,16 @@ final class CelerityStaticResourceResponse extends Phobject {
         $type));
   }
 
-  public function renderHTMLFooter() {
+  public function renderHTMLFooter($is_frameable) {
     $this->metadataLocked = true;
 
-    $data = array();
-    if ($this->metadata) {
-      $json_metadata = AphrontResponse::encodeJSONForHTTPResponse(
-        $this->metadata);
-      $this->metadata = array();
-    } else {
-      $json_metadata = '{}';
-    }
-    // Even if there is no metadata on the page, Javelin uses the mergeData()
-    // call to start dispatching the event queue.
-    $data[] = 'JX.Stratcom.mergeData('.$this->metadataBlock.', '.
-                                       $json_metadata.');';
+    $merge_data = array(
+      'block' => $this->metadataBlock,
+      'data' => $this->metadata,
+    );
+    $this->metadata = array();
 
-    $onload = array();
+    $behavior_lists = array();
     if ($this->behaviors) {
       $behaviors = $this->behaviors;
       $this->behaviors = array();
@@ -241,24 +244,52 @@ final class CelerityStaticResourceResponse extends Phobject {
         if (!$group) {
           continue;
         }
-        $group_json = AphrontResponse::encodeJSONForHTTPResponse(
-          $group);
-        $onload[] = 'JX.initBehaviors('.$group_json.')';
+        $behavior_lists[] = $group;
       }
     }
 
-    if ($onload) {
-      foreach ($onload as $func) {
-        $data[] = 'JX.onload(function(){'.$func.'});';
-      }
+    $initializers = array();
+
+    // Even if there is no metadata on the page, Javelin uses the mergeData()
+    // call to start dispatching the event queue, so we always want to include
+    // this initializer.
+    $initializers[] = array(
+      'kind' => 'merge',
+      'data' => $merge_data,
+    );
+
+    foreach ($behavior_lists as $behavior_list) {
+      $initializers[] = array(
+        'kind' => 'behaviors',
+        'data' => $behavior_list,
+      );
     }
 
-    if ($data) {
-      $data = implode("\n", $data);
-      return self::renderInlineScript($data);
-    } else {
-      return '';
+    if ($is_frameable) {
+      $initializers[] = array(
+        'data' => 'frameable',
+        'kind' => (bool)$is_frameable,
+      );
     }
+
+    $tags = array();
+    foreach ($initializers as $initializer) {
+      $data = $initializer['data'];
+      if (is_array($data)) {
+        $json_data = AphrontResponse::encodeJSONForHTTPResponse($data);
+      } else {
+        $json_data = json_encode($data);
+      }
+
+      $tags[] = phutil_tag(
+        'data',
+        array(
+          'data-javelin-init-kind' => $initializer['kind'],
+          'data-javelin-init-data' => $json_data,
+        ));
+    }
+
+    return $tags;
   }
 
   public static function renderInlineScript($data) {
diff --git a/src/applications/celerity/controller/CelerityResourceController.php b/src/applications/celerity/controller/CelerityResourceController.php
index 730e5ddc90..702fcefb2c 100644
--- a/src/applications/celerity/controller/CelerityResourceController.php
+++ b/src/applications/celerity/controller/CelerityResourceController.php
@@ -106,6 +106,11 @@ abstract class CelerityResourceController extends PhabricatorController {
     $response = id(new AphrontFileResponse())
       ->setMimeType($type_map[$type]);
 
+    // The "Content-Security-Policy" header has no effect on the actual
+    // resources, only on the main request. Disable it on the resource
+    // responses to limit confusion.
+    $response->setDisableContentSecurityPolicy(true);
+
     $range = AphrontRequest::getHTTPHeader('Range');
 
     if (strlen($range)) {
diff --git a/src/applications/phortune/provider/PhortuneStripePaymentProvider.php b/src/applications/phortune/provider/PhortuneStripePaymentProvider.php
index 5adfc62e19..370f132a1f 100644
--- a/src/applications/phortune/provider/PhortuneStripePaymentProvider.php
+++ b/src/applications/phortune/provider/PhortuneStripePaymentProvider.php
@@ -275,12 +275,18 @@ final class PhortuneStripePaymentProvider extends PhortunePaymentProvider {
     AphrontRequest $request,
     array $errors) {
 
+    $src = 'https://js.stripe.com/v2/';
+
     $ccform = id(new PhortuneCreditCardForm())
       ->setSecurityAssurance(
         pht('Payments are processed securely by Stripe.'))
       ->setUser($request->getUser())
       ->setErrors($errors)
-      ->addScript('https://js.stripe.com/v2/');
+      ->addScript($src);
+
+    CelerityAPI::getStaticResourceResponse()
+      ->addContentSecurityPolicyURI('script', $src)
+      ->addContentSecurityPolicyURI('frame', $src);
 
     Javelin::initBehavior(
       'stripe-payment-form',
diff --git a/src/view/form/control/AphrontFormRecaptchaControl.php b/src/view/form/control/AphrontFormRecaptchaControl.php
index 8c056b5bf6..b0559754e9 100644
--- a/src/view/form/control/AphrontFormRecaptchaControl.php
+++ b/src/view/form/control/AphrontFormRecaptchaControl.php
@@ -42,16 +42,24 @@ final class AphrontFormRecaptchaControl extends AphrontFormControl {
     $js = 'https://www.google.com/recaptcha/api.js';
     $pubkey = PhabricatorEnv::getEnvConfig('recaptcha.public-key');
 
-    return array(
-      phutil_tag('div', array(
-        'class'        => 'g-recaptcha',
-        'data-sitekey' => $pubkey,
-      )),
+    CelerityAPI::getStaticResourceResponse()
+      ->addContentSecurityPolicyURI('script', $js)
+      ->addContentSecurityPolicyURI('script', 'https://www.gstatic.com/')
+      ->addContentSecurityPolicyURI('frame', 'https://www.google.com/');
 
-      phutil_tag('script', array(
-        'type' => 'text/javascript',
-        'src'  => $js,
-      )),
+    return array(
+      phutil_tag(
+        'div',
+        array(
+          'class' => 'g-recaptcha',
+          'data-sitekey' => $pubkey,
+        )),
+      phutil_tag(
+        'script',
+        array(
+          'type' => 'text/javascript',
+          'src'  => $js,
+        )),
     );
   }
 }
diff --git a/src/view/page/AphrontPageView.php b/src/view/page/AphrontPageView.php
index 8f3704dca2..d64c3b8e9b 100644
--- a/src/view/page/AphrontPageView.php
+++ b/src/view/page/AphrontPageView.php
@@ -59,9 +59,15 @@ abstract class AphrontPageView extends AphrontView {
       ),
       array($body, $tail));
 
+    if (PhabricatorEnv::getEnvConfig('phabricator.developer-mode')) {
+      $data_fragment = phutil_safe_html(' data-developer-mode="1"');
+    } else {
+      $data_fragment = null;
+    }
+
     $response = hsprintf(
       '<!DOCTYPE html>'.
-      '<html>'.
+      '<html%s>'.
         '<head>'.
           '<meta charset="UTF-8" />'.
           '<title>%s</title>'.
@@ -69,6 +75,7 @@ abstract class AphrontPageView extends AphrontView {
         '</head>'.
         '%s'.
       '</html>',
+      $data_fragment,
       $title,
       $head,
       $body);
diff --git a/src/view/page/PhabricatorBarePageView.php b/src/view/page/PhabricatorBarePageView.php
index c9a3c5c5cd..12eabf11e4 100644
--- a/src/view/page/PhabricatorBarePageView.php
+++ b/src/view/page/PhabricatorBarePageView.php
@@ -59,11 +59,6 @@ class PhabricatorBarePageView extends AphrontPageView {
   }
 
   protected function getHead() {
-    $framebust = null;
-    if (!$this->getFrameable()) {
-      $framebust = '(top == self) || top.location.replace(self.location.href);';
-    }
-
     $viewport_tag = null;
     if ($this->getDeviceReady()) {
       $viewport_tag = phutil_tag(
@@ -140,9 +135,8 @@ class PhabricatorBarePageView extends AphrontPageView {
       }
     }
 
-    $developer = PhabricatorEnv::getEnvConfig('phabricator.developer-mode');
     return hsprintf(
-      '%s%s%s%s%s%s%s%s%s',
+      '%s%s%s%s%s%s%s%s',
       $viewport_tag,
       $mask_icon,
       $icon_tag_76,
@@ -150,8 +144,6 @@ class PhabricatorBarePageView extends AphrontPageView {
       $icon_tag_152,
       $favicon_tag,
       $referrer_tag,
-      CelerityStaticResourceResponse::renderInlineScript(
-        $framebust.jsprintf('window.__DEV__=%d;', ($developer ? 1 : 0))),
       $response->renderResourcesOfType('css'));
   }
 
diff --git a/src/view/page/PhabricatorStandardPageView.php b/src/view/page/PhabricatorStandardPageView.php
index 78ff716a44..266e6592ee 100644
--- a/src/view/page/PhabricatorStandardPageView.php
+++ b/src/view/page/PhabricatorStandardPageView.php
@@ -608,12 +608,15 @@ final class PhabricatorStandardPageView extends PhabricatorBarePageView
         Javelin::initBehavior(
           'aphlict-listen',
           array(
-            'websocketURI'  => (string)$client_uri,
+            'websocketURI' => (string)$client_uri,
           ) + $this->buildAphlictListenConfigData());
+
+        CelerityAPI::getStaticResourceResponse()
+          ->addContentSecurityPolicyURI('connect', $client_uri);
       }
     }
 
-    $tail[] = $response->renderHTMLFooter();
+    $tail[] = $response->renderHTMLFooter($this->getFrameable());
 
     return $tail;
   }
@@ -860,6 +863,19 @@ final class PhabricatorStandardPageView extends PhabricatorBarePageView
       $blacklist[] = $application->getQuicksandURIPatternBlacklist();
     }
 
+    // See T4340. Currently, Phortune and Auth both require pulling in external
+    // Javascript (for Stripe card management and Recaptcha, respectively).
+    // This can put us in a position where the user loads a page with a
+    // restrictive Content-Security-Policy, then uses Quicksand to navigate to
+    // a page which needs to load external scripts. For now, just blacklist
+    // these entire applications since we aren't giving up anything
+    // significant by doing so.
+
+    $blacklist[] = array(
+      '/phortune/.*',
+      '/auth/.*',
+    );
+
     return array_mergev($blacklist);
   }
 
@@ -903,9 +919,17 @@ final class PhabricatorStandardPageView extends PhabricatorBarePageView
         ->setContent($content);
     } else {
       $content = $this->render();
+
       $response = id(new AphrontWebpageResponse())
         ->setContent($content)
         ->setFrameable($this->getFrameable());
+
+      $static = CelerityAPI::getStaticResourceResponse();
+      foreach ($static->getContentSecurityPolicyURIMap() as $kind => $uris) {
+        foreach ($uris as $uri) {
+          $response->addContentSecurityPolicyURI($kind, $uri);
+        }
+      }
     }
 
     return $response;
diff --git a/webroot/rsrc/externals/javelin/core/Stratcom.js b/webroot/rsrc/externals/javelin/core/Stratcom.js
index 4bfb11d4fd..9ad2b8bbd0 100644
--- a/webroot/rsrc/externals/javelin/core/Stratcom.js
+++ b/webroot/rsrc/externals/javelin/core/Stratcom.js
@@ -517,6 +517,36 @@ JX.install('Stratcom', {
       return len ? this._execContext[len - 1].event : null;
     },
 
+    initialize: function(initializers) {
+      var frameable = false;
+
+      for (var ii = 0; ii < initializers.length; ii++) {
+        var kind = initializers[ii].kind;
+        var data = initializers[ii].data;
+        switch (kind) {
+          case 'behaviors':
+            JX.initBehaviors(data);
+            break;
+          case 'merge':
+            JX.Stratcom.mergeData(data.block, data.data);
+            JX.Stratcom.ready = true;
+            break;
+          case 'frameable':
+            frameable = !!data;
+            break;
+        }
+      }
+
+      // If the initializer tags did not explicitly allow framing, framebust.
+      // This protects us from clickjacking attacks on older versions of IE.
+      // The "X-Frame-Options" and "Content-Security-Policy" headers provide
+      // more modern variations of this protection.
+      if (!frameable) {
+        if (window.top != window.self) {
+          window.top.location.replace(window.self.location.href);
+        }
+      }
+    },
 
     /**
      * Merge metadata. You must call this (even if you have no metadata) to
@@ -542,7 +572,6 @@ JX.install('Stratcom', {
       } else {
         this._data[block] = data;
         if (block === 0) {
-          JX.Stratcom.ready = true;
           JX.flushHoldingQueue('install-init', function(fn) {
             fn();
           });
diff --git a/webroot/rsrc/externals/javelin/core/init.js b/webroot/rsrc/externals/javelin/core/init.js
index 88a9cf137d..fb10b0f4c3 100644
--- a/webroot/rsrc/externals/javelin/core/init.js
+++ b/webroot/rsrc/externals/javelin/core/init.js
@@ -46,25 +46,51 @@
   makeHoldingQueue('behavior');
   makeHoldingQueue('install-init');
 
-  window.__DEV__ = window.__DEV__ || 0;
-
   var loaded = false;
   var onload = [];
   var master_event_queue = [];
   var root = document.documentElement;
   var has_add_event_listener = !!root.addEventListener;
 
+  window.__DEV__ = !!root.getAttribute('data-developer-mode');
+
   JX.__rawEventQueue = function(what) {
     master_event_queue.push(what);
 
-    // Evade static analysis - JX.Stratcom
+    var ii;
     var Stratcom = JX['Stratcom'];
-    if (Stratcom && Stratcom.ready) {
-      //  Empty the queue now so that exceptions don't cause us to repeatedly
-      //  try to handle events.
+
+    if (!loaded && what.type == 'domready') {
+      var initializers = [];
+
+      var tags = JX.DOM.scry(document.body, 'data');
+      for (ii = 0; ii < tags.length; ii++) {
+
+        // Ignore tags which are not immediate children of the document
+        // body. If an attacker somehow injects arbitrary tags into the
+        // content of the document, that should not give them access to
+        // modify initialization behaviors.
+        if (tags[ii].parentNode !== document.body) {
+          continue;
+        }
+
+        var tag_kind = tags[ii].getAttribute('data-javelin-init-kind');
+        var tag_data = tags[ii].getAttribute('data-javelin-init-data');
+        tag_data = JX.JSON.parse(tag_data);
+
+        initializers.push({kind: tag_kind, data: tag_data});
+      }
+
+      Stratcom.initialize(initializers);
+      loaded = true;
+    }
+
+    if (loaded) {
+      // Empty the queue now so that exceptions don't cause us to repeatedly
+      // try to handle events.
       var local_queue = master_event_queue;
       master_event_queue = [];
-      for (var ii = 0; ii < local_queue.length; ++ii) {
+      for (ii = 0; ii < local_queue.length; ++ii) {
         var evt = local_queue[ii];
 
         // Sometimes IE gives us events which throw when ".type" is accessed;
@@ -72,11 +98,10 @@
         // figure out where these are coming from.
         try { var test = evt.type; } catch (x) { continue; }
 
-        if (!loaded && evt.type == 'domready') {
+        if (evt.type == 'domready') {
           // NOTE: Firefox interprets "document.body.id = null" as the string
           // literal "null".
           document.body && (document.body.id = '');
-          loaded = true;
           for (var jj = 0; jj < onload.length; jj++) {
             onload[jj]();
           }

From 9658249ac5a0ba6a44bb09d9ea53553df9aef120 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Tue, 27 Feb 2018 12:07:54 -0800
Subject: [PATCH 15/39] Add "Referrer-Policy: no-referrer" to standard HTTP
 headers

Summary:
Ref T4340. Some browsers respect this header and referrers are a plague upon the earth.

Also, upgrade "never" to the more modern value "no-referrer".

Test Plan:
In Safari, Firefox and Chrome, disabled `rel="noreferrer"` on links and generated a normal link to an external site. Then clicked it and checked if a referrer was sent.

  - Safari respects meta only, but "no-referrer" is fine.
  - Firefox respects both (either the header or meta tag are individually sufficient to stop referrers).
  - Chrome respects both (same as Firefox).

Maniphest Tasks: T4340

Differential Revision: https://secure.phabricator.com/D19144
---
 src/aphront/response/AphrontResponse.php  | 2 ++
 src/view/page/PhabricatorBarePageView.php | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/aphront/response/AphrontResponse.php b/src/aphront/response/AphrontResponse.php
index 47ec2e465b..34cbd4081f 100644
--- a/src/aphront/response/AphrontResponse.php
+++ b/src/aphront/response/AphrontResponse.php
@@ -92,6 +92,8 @@ abstract class AphrontResponse extends Phobject {
       $headers[] = array('Content-Security-Policy', $csp);
     }
 
+    $headers[] = array('Referrer-Policy', 'no-referrer');
+
     return $headers;
   }
 
diff --git a/src/view/page/PhabricatorBarePageView.php b/src/view/page/PhabricatorBarePageView.php
index 12eabf11e4..afdc482170 100644
--- a/src/view/page/PhabricatorBarePageView.php
+++ b/src/view/page/PhabricatorBarePageView.php
@@ -119,7 +119,7 @@ class PhabricatorBarePageView extends AphrontPageView {
       'meta',
       array(
         'name' => 'referrer',
-        'content' => 'never',
+        'content' => 'no-referrer',
       ));
 
     $response = CelerityAPI::getStaticResourceResponse();

From 80fe382e3d2782f56b6496899e0fe93aba2f5220 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Tue, 27 Feb 2018 13:44:53 -0800
Subject: [PATCH 16/39] Add "Committer's projects" and "Author's projects"
 fields to Herald commit rules

Summary:
Ref T13093. See PHI396. These are possibly somewhat niche, but reasonable to support and consistent with the existing "Pusher's projects".

Also relabel "Pusher's projects" and "Project tags" for consistency and, hopefully, clarity.

Test Plan:
  - Created new "commit" and "hook: commit content" Herald rules which run against "Author's projects" and "Committer's projects".
  - Test console'd the "Commit" rules.
  - Pushed through the "Hook" rule.
  - In all cases, saw fields populate appropriately.

Maniphest Tasks: T13093

Differential Revision: https://secure.phabricator.com/D19145
---
 src/__phutil_library_map__.php                |  8 ++++
 ...ffusionCommitAuthorProjectsHeraldField.php | 38 +++++++++++++++++++
 ...sionCommitCommitterProjectsHeraldField.php | 38 +++++++++++++++++++
 ...CommitContentAuthorProjectsHeraldField.php | 38 +++++++++++++++++++
 ...mitContentCommitterProjectsHeraldField.php | 38 +++++++++++++++++++
 ...CommitContentPusherProjectsHeraldField.php |  2 +-
 ...nPreCommitRefPusherProjectsHeraldField.php |  2 +-
 ...CommitRefRepositoryProjectsHeraldField.php |  2 +-
 .../project/herald/HeraldProjectsField.php    |  2 +-
 9 files changed, 164 insertions(+), 4 deletions(-)
 create mode 100644 src/applications/diffusion/herald/DiffusionCommitAuthorProjectsHeraldField.php
 create mode 100644 src/applications/diffusion/herald/DiffusionCommitCommitterProjectsHeraldField.php
 create mode 100644 src/applications/diffusion/herald/DiffusionPreCommitContentAuthorProjectsHeraldField.php
 create mode 100644 src/applications/diffusion/herald/DiffusionPreCommitContentCommitterProjectsHeraldField.php

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
index 3a7ba1dbbc..b72577583d 100644
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -652,10 +652,12 @@ phutil_register_library_map(array(
     'DiffusionCommitAuditorsHeraldField' => 'applications/diffusion/herald/DiffusionCommitAuditorsHeraldField.php',
     'DiffusionCommitAuditorsTransaction' => 'applications/diffusion/xaction/DiffusionCommitAuditorsTransaction.php',
     'DiffusionCommitAuthorHeraldField' => 'applications/diffusion/herald/DiffusionCommitAuthorHeraldField.php',
+    'DiffusionCommitAuthorProjectsHeraldField' => 'applications/diffusion/herald/DiffusionCommitAuthorProjectsHeraldField.php',
     'DiffusionCommitAutocloseHeraldField' => 'applications/diffusion/herald/DiffusionCommitAutocloseHeraldField.php',
     'DiffusionCommitBranchesController' => 'applications/diffusion/controller/DiffusionCommitBranchesController.php',
     'DiffusionCommitBranchesHeraldField' => 'applications/diffusion/herald/DiffusionCommitBranchesHeraldField.php',
     'DiffusionCommitCommitterHeraldField' => 'applications/diffusion/herald/DiffusionCommitCommitterHeraldField.php',
+    'DiffusionCommitCommitterProjectsHeraldField' => 'applications/diffusion/herald/DiffusionCommitCommitterProjectsHeraldField.php',
     'DiffusionCommitConcernTransaction' => 'applications/diffusion/xaction/DiffusionCommitConcernTransaction.php',
     'DiffusionCommitController' => 'applications/diffusion/controller/DiffusionCommitController.php',
     'DiffusionCommitDiffContentAddedHeraldField' => 'applications/diffusion/herald/DiffusionCommitDiffContentAddedHeraldField.php',
@@ -801,9 +803,11 @@ phutil_register_library_map(array(
     'DiffusionPhpExternalSymbolsSource' => 'applications/diffusion/symbol/DiffusionPhpExternalSymbolsSource.php',
     'DiffusionPreCommitContentAffectedFilesHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitContentAffectedFilesHeraldField.php',
     'DiffusionPreCommitContentAuthorHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitContentAuthorHeraldField.php',
+    'DiffusionPreCommitContentAuthorProjectsHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitContentAuthorProjectsHeraldField.php',
     'DiffusionPreCommitContentAuthorRawHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitContentAuthorRawHeraldField.php',
     'DiffusionPreCommitContentBranchesHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitContentBranchesHeraldField.php',
     'DiffusionPreCommitContentCommitterHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitContentCommitterHeraldField.php',
+    'DiffusionPreCommitContentCommitterProjectsHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitContentCommitterProjectsHeraldField.php',
     'DiffusionPreCommitContentCommitterRawHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitContentCommitterRawHeraldField.php',
     'DiffusionPreCommitContentDiffContentAddedHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitContentDiffContentAddedHeraldField.php',
     'DiffusionPreCommitContentDiffContentHeraldField' => 'applications/diffusion/herald/DiffusionPreCommitContentDiffContentHeraldField.php',
@@ -5867,10 +5871,12 @@ phutil_register_library_map(array(
     'DiffusionCommitAuditorsHeraldField' => 'DiffusionCommitHeraldField',
     'DiffusionCommitAuditorsTransaction' => 'DiffusionCommitTransactionType',
     'DiffusionCommitAuthorHeraldField' => 'DiffusionCommitHeraldField',
+    'DiffusionCommitAuthorProjectsHeraldField' => 'DiffusionCommitHeraldField',
     'DiffusionCommitAutocloseHeraldField' => 'DiffusionCommitHeraldField',
     'DiffusionCommitBranchesController' => 'DiffusionController',
     'DiffusionCommitBranchesHeraldField' => 'DiffusionCommitHeraldField',
     'DiffusionCommitCommitterHeraldField' => 'DiffusionCommitHeraldField',
+    'DiffusionCommitCommitterProjectsHeraldField' => 'DiffusionCommitHeraldField',
     'DiffusionCommitConcernTransaction' => 'DiffusionCommitAuditTransaction',
     'DiffusionCommitController' => 'DiffusionController',
     'DiffusionCommitDiffContentAddedHeraldField' => 'DiffusionCommitHeraldField',
@@ -6019,9 +6025,11 @@ phutil_register_library_map(array(
     'DiffusionPhpExternalSymbolsSource' => 'DiffusionExternalSymbolsSource',
     'DiffusionPreCommitContentAffectedFilesHeraldField' => 'DiffusionPreCommitContentHeraldField',
     'DiffusionPreCommitContentAuthorHeraldField' => 'DiffusionPreCommitContentHeraldField',
+    'DiffusionPreCommitContentAuthorProjectsHeraldField' => 'DiffusionPreCommitContentHeraldField',
     'DiffusionPreCommitContentAuthorRawHeraldField' => 'DiffusionPreCommitContentHeraldField',
     'DiffusionPreCommitContentBranchesHeraldField' => 'DiffusionPreCommitContentHeraldField',
     'DiffusionPreCommitContentCommitterHeraldField' => 'DiffusionPreCommitContentHeraldField',
+    'DiffusionPreCommitContentCommitterProjectsHeraldField' => 'DiffusionPreCommitContentHeraldField',
     'DiffusionPreCommitContentCommitterRawHeraldField' => 'DiffusionPreCommitContentHeraldField',
     'DiffusionPreCommitContentDiffContentAddedHeraldField' => 'DiffusionPreCommitContentHeraldField',
     'DiffusionPreCommitContentDiffContentHeraldField' => 'DiffusionPreCommitContentHeraldField',
diff --git a/src/applications/diffusion/herald/DiffusionCommitAuthorProjectsHeraldField.php b/src/applications/diffusion/herald/DiffusionCommitAuthorProjectsHeraldField.php
new file mode 100644
index 0000000000..a90480d50a
--- /dev/null
+++ b/src/applications/diffusion/herald/DiffusionCommitAuthorProjectsHeraldField.php
@@ -0,0 +1,38 @@
+<?php
+
+final class DiffusionCommitAuthorProjectsHeraldField
+  extends DiffusionCommitHeraldField {
+
+  const FIELDCONST = 'diffusion.commit.author.projects';
+
+  public function getHeraldFieldName() {
+    return pht("Author's projects");
+  }
+
+  public function getHeraldFieldValue($object) {
+    $adapter = $this->getAdapter();
+
+    $phid = $object->getCommitData()->getCommitDetail('authorPHID');
+    if (!$phid) {
+      return array();
+    }
+
+    $viewer = $adapter->getViewer();
+
+    $projects = id(new PhabricatorProjectQuery())
+      ->setViewer($viewer)
+      ->withMemberPHIDs(array($phid))
+      ->execute();
+
+    return mpull($projects, 'getPHID');
+  }
+
+  protected function getHeraldFieldStandardType() {
+    return self::STANDARD_PHID_LIST;
+  }
+
+  protected function getDatasource() {
+    return new PhabricatorProjectDatasource();
+  }
+
+}
diff --git a/src/applications/diffusion/herald/DiffusionCommitCommitterProjectsHeraldField.php b/src/applications/diffusion/herald/DiffusionCommitCommitterProjectsHeraldField.php
new file mode 100644
index 0000000000..97e604a481
--- /dev/null
+++ b/src/applications/diffusion/herald/DiffusionCommitCommitterProjectsHeraldField.php
@@ -0,0 +1,38 @@
+<?php
+
+final class DiffusionCommitCommitterProjectsHeraldField
+  extends DiffusionCommitHeraldField {
+
+  const FIELDCONST = 'diffusion.commit.committer.projects';
+
+  public function getHeraldFieldName() {
+    return pht("Committer's projects");
+  }
+
+  public function getHeraldFieldValue($object) {
+    $adapter = $this->getAdapter();
+
+    $phid = $object->getCommitData()->getCommitDetail('committerPHID');
+    if (!$phid) {
+      return array();
+    }
+
+    $viewer = $adapter->getViewer();
+
+    $projects = id(new PhabricatorProjectQuery())
+      ->setViewer($viewer)
+      ->withMemberPHIDs(array($phid))
+      ->execute();
+
+    return mpull($projects, 'getPHID');
+  }
+
+  protected function getHeraldFieldStandardType() {
+    return self::STANDARD_PHID_LIST;
+  }
+
+  protected function getDatasource() {
+    return new PhabricatorProjectDatasource();
+  }
+
+}
diff --git a/src/applications/diffusion/herald/DiffusionPreCommitContentAuthorProjectsHeraldField.php b/src/applications/diffusion/herald/DiffusionPreCommitContentAuthorProjectsHeraldField.php
new file mode 100644
index 0000000000..ec138c9bde
--- /dev/null
+++ b/src/applications/diffusion/herald/DiffusionPreCommitContentAuthorProjectsHeraldField.php
@@ -0,0 +1,38 @@
+<?php
+
+final class DiffusionPreCommitContentAuthorProjectsHeraldField
+  extends DiffusionPreCommitContentHeraldField {
+
+  const FIELDCONST = 'diffusion.pre.commit.author.projects';
+
+  public function getHeraldFieldName() {
+    return pht("Author's projects");
+  }
+
+  public function getHeraldFieldValue($object) {
+    $adapter = $this->getAdapter();
+
+    $phid = $adapter->getAuthorPHID();
+    if (!$phid) {
+      return array();
+    }
+
+    $viewer = $adapter->getViewer();
+
+    $projects = id(new PhabricatorProjectQuery())
+      ->setViewer($viewer)
+      ->withMemberPHIDs(array($phid))
+      ->execute();
+
+    return mpull($projects, 'getPHID');
+  }
+
+  protected function getHeraldFieldStandardType() {
+    return self::STANDARD_PHID_LIST;
+  }
+
+  protected function getDatasource() {
+    return new PhabricatorProjectDatasource();
+  }
+
+}
diff --git a/src/applications/diffusion/herald/DiffusionPreCommitContentCommitterProjectsHeraldField.php b/src/applications/diffusion/herald/DiffusionPreCommitContentCommitterProjectsHeraldField.php
new file mode 100644
index 0000000000..9b0f012f3c
--- /dev/null
+++ b/src/applications/diffusion/herald/DiffusionPreCommitContentCommitterProjectsHeraldField.php
@@ -0,0 +1,38 @@
+<?php
+
+final class DiffusionPreCommitContentCommitterProjectsHeraldField
+  extends DiffusionPreCommitContentHeraldField {
+
+  const FIELDCONST = 'diffusion.pre.commit.committer.projects';
+
+  public function getHeraldFieldName() {
+    return pht("Committer's projects");
+  }
+
+  public function getHeraldFieldValue($object) {
+    $adapter = $this->getAdapter();
+
+    $phid = $adapter->getCommitterPHID();
+    if (!$phid) {
+      return array();
+    }
+
+    $viewer = $adapter->getViewer();
+
+    $projects = id(new PhabricatorProjectQuery())
+      ->setViewer($viewer)
+      ->withMemberPHIDs(array($phid))
+      ->execute();
+
+    return mpull($projects, 'getPHID');
+  }
+
+  protected function getHeraldFieldStandardType() {
+    return self::STANDARD_PHID_LIST;
+  }
+
+  protected function getDatasource() {
+    return new PhabricatorProjectDatasource();
+  }
+
+}
diff --git a/src/applications/diffusion/herald/DiffusionPreCommitContentPusherProjectsHeraldField.php b/src/applications/diffusion/herald/DiffusionPreCommitContentPusherProjectsHeraldField.php
index 6a8c79c108..eef82124cf 100644
--- a/src/applications/diffusion/herald/DiffusionPreCommitContentPusherProjectsHeraldField.php
+++ b/src/applications/diffusion/herald/DiffusionPreCommitContentPusherProjectsHeraldField.php
@@ -6,7 +6,7 @@ final class DiffusionPreCommitContentPusherProjectsHeraldField
   const FIELDCONST = 'diffusion.pre.content.pusher.projects';
 
   public function getHeraldFieldName() {
-    return pht('Pusher projects');
+    return pht("Pusher's projects");
   }
 
   public function getHeraldFieldValue($object) {
diff --git a/src/applications/diffusion/herald/DiffusionPreCommitRefPusherProjectsHeraldField.php b/src/applications/diffusion/herald/DiffusionPreCommitRefPusherProjectsHeraldField.php
index 6fff0a3570..937e758acf 100644
--- a/src/applications/diffusion/herald/DiffusionPreCommitRefPusherProjectsHeraldField.php
+++ b/src/applications/diffusion/herald/DiffusionPreCommitRefPusherProjectsHeraldField.php
@@ -6,7 +6,7 @@ final class DiffusionPreCommitRefPusherProjectsHeraldField
   const FIELDCONST = 'diffusion.pre.ref.pusher.projects';
 
   public function getHeraldFieldName() {
-    return pht('Pusher projects');
+    return pht("Pusher's projects");
   }
 
   public function getHeraldFieldValue($object) {
diff --git a/src/applications/diffusion/herald/DiffusionPreCommitRefRepositoryProjectsHeraldField.php b/src/applications/diffusion/herald/DiffusionPreCommitRefRepositoryProjectsHeraldField.php
index df262524f2..17670a1c41 100644
--- a/src/applications/diffusion/herald/DiffusionPreCommitRefRepositoryProjectsHeraldField.php
+++ b/src/applications/diffusion/herald/DiffusionPreCommitRefRepositoryProjectsHeraldField.php
@@ -16,7 +16,7 @@ final class DiffusionPreCommitRefRepositoryProjectsHeraldField
   }
 
   protected function getHeraldFieldStandardType() {
-    return HeraldField::STANDARD_PHID_LIST;
+    return self::STANDARD_PHID_LIST;
   }
 
   protected function getDatasource() {
diff --git a/src/applications/project/herald/HeraldProjectsField.php b/src/applications/project/herald/HeraldProjectsField.php
index 1e25948689..7f66a00cf3 100644
--- a/src/applications/project/herald/HeraldProjectsField.php
+++ b/src/applications/project/herald/HeraldProjectsField.php
@@ -5,7 +5,7 @@ final class HeraldProjectsField extends HeraldField {
   const FIELDCONST = 'projects';
 
   public function getHeraldFieldName() {
-    return pht('Projects');
+    return pht('Project tags');
   }
 
   public function getFieldGroupKey() {

From 8cbfb386bb87a8d945a507a0ff16bd084878f442 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Tue, 27 Feb 2018 14:10:29 -0800
Subject: [PATCH 17/39] When evaluating the "Branches" Herald field from the
 test console, use the current viewer

Summary: Ref T13093. Depends on D19145. See PHI398. Previously, see D18933. This provides the current viewer to `ConduitCall` so that we don't try to use device credentials from unprivileged web hosts.

Test Plan: Evaluated the "Branches" field locally, saw an appropriate field value.

Maniphest Tasks: T13093

Differential Revision: https://secure.phabricator.com/D19146
---
 .../diffusion/herald/DiffusionCommitBranchesHeraldField.php   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/applications/diffusion/herald/DiffusionCommitBranchesHeraldField.php b/src/applications/diffusion/herald/DiffusionCommitBranchesHeraldField.php
index 03c4e3be62..064830ebc3 100644
--- a/src/applications/diffusion/herald/DiffusionCommitBranchesHeraldField.php
+++ b/src/applications/diffusion/herald/DiffusionCommitBranchesHeraldField.php
@@ -10,6 +10,8 @@ final class DiffusionCommitBranchesHeraldField
   }
 
   public function getHeraldFieldValue($object) {
+    $viewer = $this->getAdapter()->getViewer();
+
     $commit = $object;
     $repository = $object->getRepository();
 
@@ -19,7 +21,7 @@ final class DiffusionCommitBranchesHeraldField
     );
 
     $result = id(new ConduitCall('diffusion.branchquery', $params))
-      ->setUser(PhabricatorUser::getOmnipotentUser())
+      ->setUser($viewer)
       ->execute();
 
     $refs = DiffusionRepositoryRef::loadAllFromDictionaries($result);

From ada4f65565a0a923872206d27822e198f1a9bb00 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Tue, 27 Feb 2018 14:49:23 -0800
Subject: [PATCH 18/39] Stop sending Phriction edit mail to the
 previous/current content author

Summary:
See <https://discourse.phabricator-community.org/t/previous-author-of-phriction-page-always-receives-mail-when-new-author-edits-the-page/1155>.

After the "send users email when they are removed from the reviewer list" change which includes old To/Cc on the list, Phriction now emails the previous editor.

This is weird and unintended. The behavior isn't necessarily always bad, but if it's sometimes desirable we should make it explicit.

For now, just get rid of the weird side effect.

Test Plan: Edited a Phriction page.

Differential Revision: https://secure.phabricator.com/D19147
---
 src/applications/phriction/editor/PhrictionTransactionEditor.php | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/applications/phriction/editor/PhrictionTransactionEditor.php b/src/applications/phriction/editor/PhrictionTransactionEditor.php
index 793c609517..e6bf46c150 100644
--- a/src/applications/phriction/editor/PhrictionTransactionEditor.php
+++ b/src/applications/phriction/editor/PhrictionTransactionEditor.php
@@ -273,7 +273,6 @@ final class PhrictionTransactionEditor
 
   protected function getMailTo(PhabricatorLiskDAO $object) {
     return array(
-      $object->getContent()->getAuthorPHID(),
       $this->getActingAsPHID(),
     );
   }

From 985d499f50998011e047226cc89e4096a4311b4b Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Wed, 28 Feb 2018 05:42:55 -0800
Subject: [PATCH 19/39] Fix a Harbormaster build log issue where too few header
 lines were returned when expanding logs in the middle

Summary: Ref T13088. This variable bled through from an earlier loop and caused us to drop some of the lines in the middle.

Test Plan: Clicked "Show More", got an equal number of header and footer lines.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19148
---
 .../controller/HarbormasterBuildLogRenderController.php       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
index c1c4275982..50b17523ca 100644
--- a/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
+++ b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
@@ -167,7 +167,7 @@ final class HarbormasterBuildLogRenderController
             'Unable to find lines.'));
       }
 
-      if ($direction > 0) {
+      if ($view['direction'] > 0) {
         $slice_offset = $anchor_key;
       } else {
         $slice_offset = max(0, $anchor_key - ($view['lines'] - 1));
@@ -589,7 +589,7 @@ final class HarbormasterBuildLogRenderController
       $up_text);
 
     $mid_text = pht(
-      'Show More (%s bytes Hidden)',
+      'Show More (%s Bytes)',
       new PhutilNumber($range['tail'] - $range['head']));
 
     $expand_mid = javelin_tag(

From 143033dc1f4b2f7689c1abd5b08126fdea2049d1 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Wed, 28 Feb 2018 06:17:03 -0800
Subject: [PATCH 20/39] When showing a small piece of a Harbormaster build log,
 load a small piece of data instead of the entire log

Summary: Depends on D19148. Ref T13088. The new rendering always executes range requests for data it needs, and we can satisfy these requests by loading the smallest number of chunks which span that range.

Test Plan: Piped 50,000 lines of Apache log into Harbormaster, viewed it in the new UI, got sensible rendering times and a reasonable amount of data actually going over the wire.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19149
---
 .../autopatches/20180228.log.01.offset.sql    |   5 +
 .../storage/build/HarbormasterBuildLog.php    | 117 ++++++++++++++++--
 .../build/HarbormasterBuildLogChunk.php       |   8 +-
 3 files changed, 120 insertions(+), 10 deletions(-)
 create mode 100644 resources/sql/autopatches/20180228.log.01.offset.sql

diff --git a/resources/sql/autopatches/20180228.log.01.offset.sql b/resources/sql/autopatches/20180228.log.01.offset.sql
new file mode 100644
index 0000000000..db20fc292e
--- /dev/null
+++ b/resources/sql/autopatches/20180228.log.01.offset.sql
@@ -0,0 +1,5 @@
+ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlogchunk
+  ADD headOffset BIGINT UNSIGNED NOT NULL;
+
+ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlogchunk
+  ADD tailOffset BIGINT UNSIGNED NOT NULL;
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
index 64934bbea8..ccee7724c3 100644
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
@@ -130,7 +130,85 @@ final class HarbormasterBuildLog
   }
 
   public function loadData($offset, $length) {
-    return substr($this->getLogText(), $offset, $length);
+    $end = ($offset + $length);
+
+    $chunks = id(new HarbormasterBuildLogChunk())->loadAllWhere(
+      'logID = %d AND headOffset < %d AND tailOffset >= %d
+        ORDER BY headOffset ASC',
+      $this->getID(),
+      $end,
+      $offset);
+
+    // Make sure that whatever we read out of the database is a single
+    // contiguous range which contains all of the requested bytes.
+    $ranges = array();
+    foreach ($chunks as $chunk) {
+      $ranges[] = array(
+        'head' => $chunk->getHeadOffset(),
+        'tail' => $chunk->getTailOffset(),
+      );
+    }
+
+    $ranges = isort($ranges, 'head');
+    $ranges = array_values($ranges);
+    $count = count($ranges);
+    for ($ii = 0; $ii < ($count - 1); $ii++) {
+      if ($ranges[$ii + 1]['head'] === $ranges[$ii]['tail']) {
+        $ranges[$ii + 1]['head'] = $ranges[$ii]['head'];
+        unset($ranges[$ii]);
+      }
+    }
+
+    if (count($ranges) !== 1) {
+      $display_ranges = array();
+      foreach ($ranges as $range) {
+        $display_ranges[] = pht(
+          '(%d - %d)',
+          $range['head'],
+          $range['tail']);
+      }
+
+      if (!$display_ranges) {
+        $display_ranges[] = pht('<null>');
+      }
+
+      throw new Exception(
+        pht(
+          'Attempt to load log bytes (%d - %d) failed: failed to '.
+          'load a single contiguous range. Actual ranges: %s.',
+          implode('; ', $display_ranges)));
+    }
+
+    $range = head($ranges);
+    if ($range['head'] > $offset || $range['tail'] < $end) {
+      throw new Exception(
+        pht(
+          'Attempt to load log bytes (%d - %d) failed: the loaded range '.
+          '(%d - %d) does not span the requested range.',
+          $offset,
+          $end,
+          $range['head'],
+          $range['tail']));
+    }
+
+    $parts = array();
+    foreach ($chunks as $chunk) {
+      $parts[] = $chunk->getChunkDisplayText();
+    }
+    $parts = implode('', $parts);
+
+    $chop_head = ($offset - $range['head']);
+    $chop_tail = ($range['tail'] - $end);
+
+    if ($chop_head) {
+      $parts = substr($parts, $chop_head);
+    }
+
+    if ($chop_tail) {
+      $parts = substr($parts, 0, -$chop_tail);
+    }
+
+    return $parts;
   }
 
   public function getReadPosition($read_offset) {
@@ -220,17 +298,18 @@ final class HarbormasterBuildLog
 
     $this->openTransaction();
 
+    $offset = 0;
     foreach ($chunks as $chunk) {
       $rope->append($chunk->getChunkDisplayText());
       $chunk->delete();
 
       while ($rope->getByteLength() > $byte_limit) {
-        $this->writeEncodedChunk($rope, $byte_limit, $mode);
+        $offset += $this->writeEncodedChunk($rope, $offset, $byte_limit, $mode);
       }
     }
 
     while ($rope->getByteLength()) {
-      $this->writeEncodedChunk($rope, $byte_limit, $mode);
+      $offset += $this->writeEncodedChunk($rope, $offset, $byte_limit, $mode);
     }
 
     $this
@@ -240,7 +319,12 @@ final class HarbormasterBuildLog
     $this->saveTransaction();
   }
 
-  private function writeEncodedChunk(PhutilRope $rope, $length, $mode) {
+  private function writeEncodedChunk(
+    PhutilRope $rope,
+    $offset,
+    $length,
+    $mode) {
+
     $data = $rope->getPrefixBytes($length);
     $size = strlen($data);
 
@@ -258,15 +342,22 @@ final class HarbormasterBuildLog
         throw new Exception(pht('Unknown chunk encoding "%s"!', $mode));
     }
 
-    $this->writeChunk($mode, $size, $data);
+    $this->writeChunk($mode, $offset, $size, $data);
 
     $rope->removeBytesFromHead($size);
+
+    return $size;
   }
 
-  private function writeChunk($encoding, $raw_size, $data) {
+  private function writeChunk($encoding, $offset, $raw_size, $data) {
+    $head_offset = $offset;
+    $tail_offset = $offset + $raw_size;
+
     return id(new HarbormasterBuildLogChunk())
       ->setLogID($this->getID())
       ->setEncoding($encoding)
+      ->setHeadOffset($head_offset)
+      ->setTailOffset($tail_offset)
       ->setSize($raw_size)
       ->setChunk($data)
       ->save();
@@ -397,13 +488,23 @@ final class HarbormasterBuildLog
         if ($append_id) {
           queryfx(
             $conn_w,
-            'UPDATE %T SET chunk = CONCAT(chunk, %B), size = %d WHERE id = %d',
+            'UPDATE %T SET
+                chunk = CONCAT(chunk, %B),
+                size = %d,
+                tailOffset = headOffset + %d,
+              WHERE
+                id = %d',
             $chunk_table,
             $append_data,
             $prefix_size + $data_size,
+            $prefix_size + $data_size,
             $append_id);
         } else {
-          $this->writeChunk($encoding_text, $data_size, $append_data);
+          $this->writeChunk(
+            $encoding_text,
+            $this->getByteLength(),
+            $data_size,
+            $append_data);
         }
 
         $this->updateLineMap($append_data);
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php
index e69bad4caf..ff3e817c58 100644
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php
@@ -5,6 +5,8 @@ final class HarbormasterBuildLogChunk
 
   protected $logID;
   protected $encoding;
+  protected $headOffset;
+  protected $tailOffset;
   protected $size;
   protected $chunk;
 
@@ -20,6 +22,8 @@ final class HarbormasterBuildLogChunk
       self::CONFIG_COLUMN_SCHEMA => array(
         'logID' => 'id',
         'encoding' => 'text32',
+        'headOffset' => 'uint64',
+        'tailOffset' => 'uint64',
 
         // T6203/NULLABILITY
         // Both the type and nullability of this column are crazily wrong.
@@ -28,8 +32,8 @@ final class HarbormasterBuildLogChunk
         'chunk' => 'bytes',
       ),
       self::CONFIG_KEY_SCHEMA => array(
-        'key_log' => array(
-          'columns' => array('logID'),
+        'key_offset' => array(
+          'columns' => array('logID', 'headOffset', 'tailOffset'),
         ),
       ),
     ) + parent::getConfiguration();

From dc6a66f7f42e0632bba055a1b0fde8e3f253b70f Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Wed, 28 Feb 2018 06:39:30 -0800
Subject: [PATCH 21/39] Add a "(prototype)" link to the standalone build log on
 build pages

Summary: Depends on D19149. Ref T13088. Since the new log requires a bunch of log reprocessing, the cutover is going to require at least some time for installs to run migrations. Add a link in the UI to ease the transition, smooth over some behaviors a little, and fix a fetch issue where we'd request past the end of the log (since this is now enforced).

Test Plan: Viewed a traditional Harbormaster build, saw links to the new standalone log pages.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19150
---
 .../HarbormasterBuildLogRenderController.php  |  8 ++-
 .../HarbormasterBuildLogViewController.php    |  6 ++
 .../HarbormasterBuildViewController.php       |  7 +++
 .../view/HarbormasterBuildLogView.php         | 55 ++++++++++++-------
 4 files changed, 55 insertions(+), 21 deletions(-)

diff --git a/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
index 50b17523ca..4b36ba83d9 100644
--- a/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
+++ b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
@@ -105,7 +105,13 @@ final class HarbormasterBuildLogRenderController
     $reads = $this->mergeOverlappingReads($reads);
 
     foreach ($reads as $key => $read) {
-      $data = $log->loadData($read['fetchOffset'], $read['fetchLength']);
+      $fetch_offset = $read['fetchOffset'];
+      $fetch_length = $read['fetchLength'];
+      if ($fetch_offset + $fetch_length > $log_size) {
+        $fetch_length = $log_size - $fetch_offset;
+      }
+
+      $data = $log->loadData($fetch_offset, $fetch_length);
 
       $offset = $read['fetchOffset'];
       $line = $read['fetchLine'];
diff --git a/src/applications/harbormaster/controller/HarbormasterBuildLogViewController.php b/src/applications/harbormaster/controller/HarbormasterBuildLogViewController.php
index 41f2aef8e5..40641795c5 100644
--- a/src/applications/harbormaster/controller/HarbormasterBuildLogViewController.php
+++ b/src/applications/harbormaster/controller/HarbormasterBuildLogViewController.php
@@ -16,6 +16,9 @@ final class HarbormasterBuildLogViewController
       return new Aphront404Response();
     }
 
+    $target = $log->getBuildTarget();
+    $build = $target->getBuild();
+
     $page_title = pht('Build Log %d', $log->getID());
 
     $log_view = id(new HarbormasterBuildLogView())
@@ -25,6 +28,9 @@ final class HarbormasterBuildLogViewController
 
     $crumbs = $this->buildApplicationCrumbs()
       ->addTextCrumb(pht('Build Logs'))
+      ->addTextCrumb(
+        pht('Build %d', $build->getID()),
+        $build->getURI())
       ->addTextCrumb($page_title)
       ->setBorder(true);
 
diff --git a/src/applications/harbormaster/controller/HarbormasterBuildViewController.php b/src/applications/harbormaster/controller/HarbormasterBuildViewController.php
index 4507139d47..0e09b4d658 100644
--- a/src/applications/harbormaster/controller/HarbormasterBuildViewController.php
+++ b/src/applications/harbormaster/controller/HarbormasterBuildViewController.php
@@ -363,12 +363,19 @@ final class HarbormasterBuildViewController
       $log_view->setLines($lines);
       $log_view->setStart($start);
 
+      $prototype_view = id(new PHUIButtonView())
+        ->setTag('a')
+        ->setHref($log->getURI())
+        ->setIcon('fa-file-text-o')
+        ->setText(pht('New View (Prototype)'));
+
       $header = id(new PHUIHeaderView())
         ->setHeader(pht(
           'Build Log %d (%s - %s)',
           $log->getID(),
           $log->getLogSource(),
           $log->getLogType()))
+        ->addActionLink($prototype_view)
         ->setSubheader($this->createLogHeader($build, $log))
         ->setUser($viewer);
 
diff --git a/src/applications/harbormaster/view/HarbormasterBuildLogView.php b/src/applications/harbormaster/view/HarbormasterBuildLogView.php
index 81705f3f49..e240aab510 100644
--- a/src/applications/harbormaster/view/HarbormasterBuildLogView.php
+++ b/src/applications/harbormaster/view/HarbormasterBuildLogView.php
@@ -44,28 +44,43 @@ final class HarbormasterBuildLogView extends AphrontView {
 
     $header->addActionLink($download_button);
 
-    $content_id = celerity_generate_unique_node_id();
-    $content_div = javelin_tag(
-      'div',
-      array(
-        'id' => $content_id,
-        'class' => 'harbormaster-log-view-loading',
-      ),
-      pht('Loading...'));
-
-    require_celerity_resource('harbormaster-css');
-
-    Javelin::initBehavior(
-      'harbormaster-log',
-      array(
-        'contentNodeID' => $content_id,
-        'renderURI' => $log->getRenderURI($this->getHighlightedLineRange()),
-      ));
-
     $box_view = id(new PHUIObjectBoxView())
       ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
-      ->setHeader($header)
-      ->appendChild($content_div);
+      ->setHeader($header);
+
+    $has_linemap = $log->getLineMap();
+    if ($has_linemap) {
+      $content_id = celerity_generate_unique_node_id();
+      $content_div = javelin_tag(
+        'div',
+        array(
+          'id' => $content_id,
+          'class' => 'harbormaster-log-view-loading',
+        ),
+        pht('Loading...'));
+
+      require_celerity_resource('harbormaster-css');
+
+      Javelin::initBehavior(
+        'harbormaster-log',
+        array(
+          'contentNodeID' => $content_id,
+          'renderURI' => $log->getRenderURI($this->getHighlightedLineRange()),
+        ));
+
+      $box_view->appendChild($content_div);
+    } else {
+      $box_view->setFormErrors(
+        array(
+          pht(
+            'This older log is missing required rendering data. To rebuild '.
+            'rendering data, run: %s',
+            phutil_tag(
+              'tt',
+              array(),
+              '$ bin/harbormaster rebuild-log --force --id '.$log->getID())),
+        ));
+    }
 
     return $box_view;
   }

From 5a2213ef82c7bbe0da6c22ee326a8d0294321396 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Wed, 28 Feb 2018 06:59:16 -0800
Subject: [PATCH 22/39] Provide API read access to Harbormaster build logs

Summary:
Depends on D19150. Ref T13088. Allow clients to retrieve information about build logs, including log data, over the API.

(To fetch log data, take the `filePHID` to `file.search`, then issue a normal GET against the URI. Use a `Content-Range` header to get part of the log.)

Test Plan: Ran `harbormaster.log.search`, got sensible-looking results.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19151
---
 src/__phutil_library_map__.php                |  5 ++
 ...ormasterBuildLogSearchConduitAPIMethod.php | 18 +++++
 .../query/HarbormasterBuildLogQuery.php       | 37 ++++-----
 .../HarbormasterBuildLogSearchEngine.php      | 79 +++++++++++++++++++
 .../storage/build/HarbormasterBuildLog.php    | 35 +++++++-
 src/view/control/AphrontTableView.php         |  2 +-
 6 files changed, 151 insertions(+), 25 deletions(-)
 create mode 100644 src/applications/harbormaster/conduit/HarbormasterBuildLogSearchConduitAPIMethod.php
 create mode 100644 src/applications/harbormaster/query/HarbormasterBuildLogSearchEngine.php

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
index b72577583d..76bc79498e 100644
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -1235,6 +1235,8 @@ phutil_register_library_map(array(
     'HarbormasterBuildLogPHIDType' => 'applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php',
     'HarbormasterBuildLogQuery' => 'applications/harbormaster/query/HarbormasterBuildLogQuery.php',
     'HarbormasterBuildLogRenderController' => 'applications/harbormaster/controller/HarbormasterBuildLogRenderController.php',
+    'HarbormasterBuildLogSearchConduitAPIMethod' => 'applications/harbormaster/conduit/HarbormasterBuildLogSearchConduitAPIMethod.php',
+    'HarbormasterBuildLogSearchEngine' => 'applications/harbormaster/query/HarbormasterBuildLogSearchEngine.php',
     'HarbormasterBuildLogTestCase' => 'applications/harbormaster/__tests__/HarbormasterBuildLogTestCase.php',
     'HarbormasterBuildLogView' => 'applications/harbormaster/view/HarbormasterBuildLogView.php',
     'HarbormasterBuildLogViewController' => 'applications/harbormaster/controller/HarbormasterBuildLogViewController.php',
@@ -6522,6 +6524,7 @@ phutil_register_library_map(array(
       'HarbormasterDAO',
       'PhabricatorPolicyInterface',
       'PhabricatorDestructibleInterface',
+      'PhabricatorConduitResultInterface',
     ),
     'HarbormasterBuildLogChunk' => 'HarbormasterDAO',
     'HarbormasterBuildLogChunkIterator' => 'PhutilBufferedIterator',
@@ -6529,6 +6532,8 @@ phutil_register_library_map(array(
     'HarbormasterBuildLogPHIDType' => 'PhabricatorPHIDType',
     'HarbormasterBuildLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
     'HarbormasterBuildLogRenderController' => 'HarbormasterController',
+    'HarbormasterBuildLogSearchConduitAPIMethod' => 'PhabricatorSearchEngineAPIMethod',
+    'HarbormasterBuildLogSearchEngine' => 'PhabricatorApplicationSearchEngine',
     'HarbormasterBuildLogTestCase' => 'PhabricatorTestCase',
     'HarbormasterBuildLogView' => 'AphrontView',
     'HarbormasterBuildLogViewController' => 'HarbormasterController',
diff --git a/src/applications/harbormaster/conduit/HarbormasterBuildLogSearchConduitAPIMethod.php b/src/applications/harbormaster/conduit/HarbormasterBuildLogSearchConduitAPIMethod.php
new file mode 100644
index 0000000000..e24b53956f
--- /dev/null
+++ b/src/applications/harbormaster/conduit/HarbormasterBuildLogSearchConduitAPIMethod.php
@@ -0,0 +1,18 @@
+<?php
+
+final class HarbormasterBuildLogSearchConduitAPIMethod
+  extends PhabricatorSearchEngineAPIMethod {
+
+  public function getAPIMethodName() {
+    return 'harbormaster.log.search';
+  }
+
+  public function newSearchEngine() {
+    return new HarbormasterBuildLogSearchEngine();
+  }
+
+  public function getMethodSummary() {
+    return pht('Find out information about build logs.');
+  }
+
+}
diff --git a/src/applications/harbormaster/query/HarbormasterBuildLogQuery.php b/src/applications/harbormaster/query/HarbormasterBuildLogQuery.php
index 3d45de686f..ad27d09582 100644
--- a/src/applications/harbormaster/query/HarbormasterBuildLogQuery.php
+++ b/src/applications/harbormaster/query/HarbormasterBuildLogQuery.php
@@ -23,19 +23,12 @@ final class HarbormasterBuildLogQuery
     return $this;
   }
 
+  public function newResultObject() {
+    return new HarbormasterBuildLog();
+  }
+
   protected function loadPage() {
-    $table = new HarbormasterBuildLog();
-    $conn_r = $table->establishConnection('r');
-
-    $data = queryfx_all(
-      $conn_r,
-      'SELECT * FROM %T %Q %Q %Q',
-      $table->getTableName(),
-      $this->buildWhereClause($conn_r),
-      $this->buildOrderClause($conn_r),
-      $this->buildLimitClause($conn_r));
-
-    return $table->loadAllFromArray($data);
+    return $this->loadStandardPage($this->newResultObject());
   }
 
   protected function willFilterPage(array $page) {
@@ -63,33 +56,31 @@ final class HarbormasterBuildLogQuery
     return $page;
   }
 
-  protected function buildWhereClause(AphrontDatabaseConnection $conn_r) {
-    $where = array();
+  protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) {
+    $where = parent::buildWhereClauseParts($conn);
 
-    if ($this->ids) {
+    if ($this->ids !== null) {
       $where[] = qsprintf(
-        $conn_r,
+        $conn,
         'id IN (%Ld)',
         $this->ids);
     }
 
-    if ($this->phids) {
+    if ($this->phids !== null) {
       $where[] = qsprintf(
-        $conn_r,
+        $conn,
         'phid IN (%Ls)',
         $this->phids);
     }
 
-    if ($this->buildTargetPHIDs) {
+    if ($this->buildTargetPHIDs !== null) {
       $where[] = qsprintf(
-        $conn_r,
+        $conn,
         'buildTargetPHID IN (%Ls)',
         $this->buildTargetPHIDs);
     }
 
-    $where[] = $this->buildPagingClause($conn_r);
-
-    return $this->formatWhereClause($where);
+    return $where;
   }
 
   public function getQueryApplicationClass() {
diff --git a/src/applications/harbormaster/query/HarbormasterBuildLogSearchEngine.php b/src/applications/harbormaster/query/HarbormasterBuildLogSearchEngine.php
new file mode 100644
index 0000000000..3698238b12
--- /dev/null
+++ b/src/applications/harbormaster/query/HarbormasterBuildLogSearchEngine.php
@@ -0,0 +1,79 @@
+<?php
+
+final class HarbormasterBuildLogSearchEngine
+  extends PhabricatorApplicationSearchEngine {
+
+  public function getResultTypeDescription() {
+    return pht('Harbormaster Build Logs');
+  }
+
+  public function getApplicationClassName() {
+    return 'PhabricatorHarbormasterApplication';
+  }
+
+  public function newQuery() {
+    return new HarbormasterBuildLogQuery();
+  }
+
+  protected function buildCustomSearchFields() {
+    return array(
+      id(new PhabricatorPHIDsSearchField())
+        ->setLabel(pht('Build Targets'))
+        ->setKey('buildTargetPHIDs')
+        ->setAliases(
+          array(
+            'buildTargetPHID',
+            'buildTargets',
+            'buildTarget',
+            'targetPHIDs',
+            'targetPHID',
+            'targets',
+            'target',
+          ))
+        ->setDescription(
+          pht('Search for logs that belong to a particular build target.')),
+    );
+  }
+
+  protected function buildQueryFromParameters(array $map) {
+    $query = $this->newQuery();
+
+    if ($map['buildTargetPHIDs']) {
+      $query->withBuildTargetPHIDs($map['buildTargetPHIDs']);
+    }
+
+    return $query;
+  }
+
+  protected function getURI($path) {
+    return '/harbormaster/log/'.$path;
+  }
+
+  protected function getBuiltinQueryNames() {
+    return array(
+      'all' => pht('All Builds'),
+    );
+  }
+
+  public function buildSavedQueryFromBuiltin($query_key) {
+    $query = $this->newSavedQuery();
+    $query->setQueryKey($query_key);
+
+    switch ($query_key) {
+      case 'all':
+        return $query;
+    }
+
+    return parent::buildSavedQueryFromBuiltin($query_key);
+  }
+
+  protected function renderResultList(
+    array $builds,
+    PhabricatorSavedQuery $query,
+    array $handles) {
+
+    // For now, this SearchEngine is only for driving the API.
+    throw new PhutilMethodNotImplementedException();
+  }
+
+}
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
index ccee7724c3..9d77abf4a8 100644
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
@@ -4,7 +4,8 @@ final class HarbormasterBuildLog
   extends HarbormasterDAO
   implements
     PhabricatorPolicyInterface,
-    PhabricatorDestructibleInterface {
+    PhabricatorDestructibleInterface,
+    PhabricatorConduitResultInterface {
 
   protected $buildTargetPHID;
   protected $logSource;
@@ -714,4 +715,36 @@ final class HarbormasterBuildLog
   }
 
 
+/* -(  PhabricatorConduitResultInterface  )---------------------------------- */
+
+
+  public function getFieldSpecificationsForConduit() {
+    return array(
+      id(new PhabricatorConduitSearchFieldSpecification())
+        ->setKey('buildTargetPHID')
+        ->setType('phid')
+        ->setDescription(pht('Build target this log is attached to.')),
+      id(new PhabricatorConduitSearchFieldSpecification())
+        ->setKey('byteLength')
+        ->setType('int')
+        ->setDescription(pht('Length of the log in bytes.')),
+      id(new PhabricatorConduitSearchFieldSpecification())
+        ->setKey('filePHID')
+        ->setType('phid?')
+        ->setDescription(pht('A file containing the log data.')),
+    );
+  }
+
+  public function getFieldValuesForConduit() {
+    return array(
+      'buildTargetPHID' => $this->getBuildTargetPHID(),
+      'byteLength' => (int)$this->getByteLength(),
+      'filePHID' => $this->getFilePHID(),
+    );
+  }
+
+  public function getConduitSearchAttachments() {
+    return array();
+  }
+
 }
diff --git a/src/view/control/AphrontTableView.php b/src/view/control/AphrontTableView.php
index 3cca7f89e5..cae9dabec2 100644
--- a/src/view/control/AphrontTableView.php
+++ b/src/view/control/AphrontTableView.php
@@ -248,7 +248,7 @@ final class AphrontTableView extends AphrontView {
 
     foreach ($col_classes as $key => $value) {
 
-      if (($sort_values[$key] !== null) &&
+      if (isset($sort_values[$key]) &&
           ($sort_values[$key] == $this->sortSelected)) {
         $value = trim($value.' sorted-column');
       }

From 21ddfe442e67bf1efebb142ec6d429b748c00c28 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Wed, 28 Feb 2018 07:43:14 -0800
Subject: [PATCH 23/39] Add a "--rate" flag to `bin/harbormaster write-log` to
 support testing live log streaming

Summary: Depends on D19151. Ref T13088. While dramatically less exciting than using `lolcat` and less general than `pv`, this should do the job adequately.

Test Plan: Piped a sizable log into `bin/harbormaster write-log` with `--rate 2048`, saw a progress bar. Loaded the log in the web UI and saw it grow as the page reloaded.

Reviewers: yelirekim

Reviewed By: yelirekim

Subscribers: yelirekim, PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19152
---
 ...HarbormasterManagementWriteLogWorkflow.php | 39 ++++++++++++++++++-
 .../storage/build/HarbormasterBuildLog.php    |  2 +-
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php b/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
index b680b382aa..3676b1415c 100644
--- a/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
+++ b/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php
@@ -18,6 +18,13 @@ final class HarbormasterManagementWriteLogWorkflow
             'param' => 'id',
             'help' => pht('Build Target ID to attach the log to.'),
           ),
+          array(
+            'name' => 'rate',
+            'param' => 'bytes',
+            'help' => pht(
+              'Limit the rate at which the log is written, to test '.
+              'live log streaming.'),
+          ),
         ));
   }
 
@@ -54,7 +61,37 @@ final class HarbormasterManagementWriteLogWorkflow
       pht('Reading log content from stdin...'));
 
     $content = file_get_contents('php://stdin');
-    $log->append($content);
+
+    $rate = $args->getArg('rate');
+    if ($rate) {
+      if ($rate <= 0) {
+        throw new Exception(
+          pht(
+            'Write rate must be more than 0 bytes/sec.'));
+      }
+
+      echo tsprintf(
+        "%s\n",
+        pht('Writing log, slowly...'));
+
+      $offset = 0;
+      $total = strlen($content);
+      $pieces = str_split($content, $rate);
+
+      $bar = id(new PhutilConsoleProgressBar())
+        ->setTotal($total);
+
+      foreach ($pieces as $piece) {
+        $log->append($piece);
+        $bar->update(strlen($piece));
+        sleep(1);
+      }
+
+      $bar->done();
+
+    } else {
+      $log->append($content);
+    }
 
     echo tsprintf(
       "%s\n",
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
index 9d77abf4a8..bf69457254 100644
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
@@ -492,7 +492,7 @@ final class HarbormasterBuildLog
             'UPDATE %T SET
                 chunk = CONCAT(chunk, %B),
                 size = %d,
-                tailOffset = headOffset + %d,
+                tailOffset = headOffset + %d
               WHERE
                 id = %d',
             $chunk_table,

From f114b2dd7d4bc6ad56f9edbfacfe4834a6ccb39a Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Wed, 28 Feb 2018 11:36:22 -0800
Subject: [PATCH 24/39] When viewing a live build log, trap users in a small
 personal hell where nothing but slavish devotion to the log exists

Summary: Depends on D19152. Ref T13088. This adds live log tailing. It is probably not the final version of this feature because it prevents escape once you begin tailing a log.

Test Plan: Used `bin/harbormaster write-log --rate ...` to write a log slowly. Viewed it in the web UI. Clicked "Follow Log". Followed the log until the write finished, a lifetime later.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19153
---
 resources/celerity/map.php                    | 14 ++--
 .../HarbormasterBuildLogRenderController.php  | 74 ++++++++++++++++---
 .../application/harbormaster/harbormaster.css |  9 ++-
 .../harbormaster/behavior-harbormaster-log.js | 48 +++++++++++-
 4 files changed, 125 insertions(+), 20 deletions(-)

diff --git a/resources/celerity/map.php b/resources/celerity/map.php
index 1f3884f8da..a59db7ed05 100644
--- a/resources/celerity/map.php
+++ b/resources/celerity/map.php
@@ -78,7 +78,7 @@ return array(
     'rsrc/css/application/feed/feed.css' => 'ecd4ec57',
     'rsrc/css/application/files/global-drag-and-drop.css' => 'b556a948',
     'rsrc/css/application/flag/flag.css' => 'bba8f811',
-    'rsrc/css/application/harbormaster/harbormaster.css' => 'c7e29d9e',
+    'rsrc/css/application/harbormaster/harbormaster.css' => '5dd4c2de',
     'rsrc/css/application/herald/herald-test.css' => 'a52e323e',
     'rsrc/css/application/herald/herald.css' => 'cd8d0134',
     'rsrc/css/application/maniphest/report.css' => '9b9580b7',
@@ -416,7 +416,7 @@ return array(
     'rsrc/js/application/drydock/drydock-live-operation-status.js' => '901935ef',
     'rsrc/js/application/files/behavior-icon-composer.js' => '8499b6ab',
     'rsrc/js/application/files/behavior-launch-icon-composer.js' => '48086888',
-    'rsrc/js/application/harbormaster/behavior-harbormaster-log.js' => 'be6974cc',
+    'rsrc/js/application/harbormaster/behavior-harbormaster-log.js' => '796a8803',
     'rsrc/js/application/herald/HeraldRuleEditor.js' => 'dca75c0e',
     'rsrc/js/application/herald/PathTypeahead.js' => 'f7fc67ec',
     'rsrc/js/application/herald/herald-rule-editor.js' => '7ebaeed3',
@@ -579,7 +579,7 @@ return array(
     'font-fontawesome' => 'e838e088',
     'font-lato' => 'c7ccd872',
     'global-drag-and-drop-css' => 'b556a948',
-    'harbormaster-css' => 'c7e29d9e',
+    'harbormaster-css' => '5dd4c2de',
     'herald-css' => 'cd8d0134',
     'herald-rule-editor' => 'dca75c0e',
     'herald-test-css' => 'a52e323e',
@@ -636,7 +636,7 @@ return array(
     'javelin-behavior-event-all-day' => 'b41537c9',
     'javelin-behavior-fancy-datepicker' => 'ecf4e799',
     'javelin-behavior-global-drag-and-drop' => '960f6a39',
-    'javelin-behavior-harbormaster-log' => 'be6974cc',
+    'javelin-behavior-harbormaster-log' => '796a8803',
     'javelin-behavior-herald-rule-editor' => '7ebaeed3',
     'javelin-behavior-high-security-warning' => 'a464fe03',
     'javelin-behavior-history-install' => '7ee2b591',
@@ -1535,6 +1535,9 @@ return array(
       'javelin-behavior',
       'javelin-quicksand',
     ),
+    '796a8803' => array(
+      'javelin-behavior',
+    ),
     '7a68dda3' => array(
       'owners-path-editor',
       'javelin-behavior',
@@ -1889,9 +1892,6 @@ return array(
       'javelin-util',
       'javelin-request',
     ),
-    'be6974cc' => array(
-      'javelin-behavior',
-    ),
     'bea6e7f4' => array(
       'javelin-install',
       'javelin-dom',
diff --git a/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
index 4b36ba83d9..6d623f2bc5 100644
--- a/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
+++ b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
@@ -22,14 +22,14 @@ final class HarbormasterBuildLogRenderController
     if ($head_lines === null) {
       $head_lines = 8;
     }
-    $head_lines = min($head_lines, 100);
+    $head_lines = min($head_lines, 1024);
     $head_lines = max($head_lines, 0);
 
     $tail_lines = $request->getInt('tail');
     if ($tail_lines === null) {
       $tail_lines = 16;
     }
-    $tail_lines = min($tail_lines, 100);
+    $tail_lines = min($tail_lines, 1024);
     $tail_lines = max($tail_lines, 0);
 
     $head_offset = $request->getInt('headOffset');
@@ -301,7 +301,6 @@ final class HarbormasterBuildLogRenderController
       );
     }
 
-
     foreach ($views as $view) {
       if ($spacer) {
         $spacer['tail'] = $view['viewOffset'];
@@ -360,6 +359,22 @@ final class HarbormasterBuildLogRenderController
       }
     }
 
+    if ($log->getLive()) {
+      $last_view = last($views);
+      $last_line = last($last_view['viewData']);
+      if ($last_line) {
+        $last_offset = $last_line['offset'];
+      } else {
+        $last_offset = 0;
+      }
+
+      $last_tail = $last_view['viewOffset'] + $last_view['viewLength'];
+      $show_live = ($last_tail === $log_size);
+      if ($show_live) {
+        $rows[] = $this->renderLiveRow($last_offset);
+      }
+    }
+
     $table = phutil_tag(
       'table',
       array(
@@ -650,25 +665,66 @@ final class HarbormasterBuildLogRenderController
         ),
         $expand_down),
     );
-    $expand_row = phutil_tag('tr', array(), $expand_cells);
-    $expand_table = phutil_tag(
+
+    return $this->renderActionTable($expand_cells);
+  }
+
+  private function renderLiveRow($log_size) {
+    $icon_down = id(new PHUIIconView())
+      ->setIcon('fa-chevron-down');
+
+    $follow = javelin_tag(
+      'a',
+      array(
+        'sigil' => 'harbormaster-log-expand harbormaster-log-live',
+        'meta' => array(
+          'headOffset' => $log_size,
+          'head' => 0,
+          'tail' => 1024,
+          'live' => true,
+        ),
+      ),
+      array(
+        $icon_down,
+        ' ',
+        pht('Follow Log'),
+        ' ',
+        $icon_down,
+      ));
+
+    $expand_cells = array(
+      phutil_tag(
+        'td',
+        array(
+          'class' => 'harbormaster-log-follow',
+        ),
+        $follow),
+    );
+
+    return $this->renderActionTable($expand_cells);
+  }
+
+  private function renderActionTable(array $action_cells) {
+    $action_row = phutil_tag('tr', array(), $action_cells);
+
+    $action_table = phutil_tag(
       'table',
       array(
         'class' => 'harbormaster-log-expand-table',
       ),
-      $expand_row);
+      $action_row);
 
-    $cells = array(
+    $format_cells = array(
       phutil_tag('th', array()),
       phutil_tag(
         'td',
         array(
           'class' => 'harbormaster-log-expand-cell',
         ),
-        $expand_table),
+        $action_table),
     );
 
-    return phutil_tag('tr', array(), $cells);
+    return phutil_tag('tr', array(), $format_cells);
   }
 
 }
diff --git a/webroot/rsrc/css/application/harbormaster/harbormaster.css b/webroot/rsrc/css/application/harbormaster/harbormaster.css
index 1c389a4383..b22658219a 100644
--- a/webroot/rsrc/css/application/harbormaster/harbormaster.css
+++ b/webroot/rsrc/css/application/harbormaster/harbormaster.css
@@ -97,7 +97,6 @@
     'Helvetica Neue', Helvetica, Arial, sans-serif;
 }
 
-
 .harbormaster-log-expand-up {
   text-align: right;
   width: 50%;
@@ -107,6 +106,14 @@
   margin: 0 0 0px 4px;
 }
 
+.harbormaster-log-follow {
+  text-align: center;
+}
+
+.harbormaster-log-follow .phui-icon-view {
+  margin: 0 4px;
+}
+
 .harbormaster-log-expand-mid {
   text-align: center;
   white-space: nowrap;
diff --git a/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js b/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js
index 38baca6199..510631e1e0 100644
--- a/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js
+++ b/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js
@@ -5,6 +5,7 @@
 
 JX.behavior('harbormaster-log', function(config) {
   var contentNode = JX.$(config.contentNodeID);
+  var following = false;
 
   JX.DOM.listen(contentNode, 'click', 'harbormaster-log-expand', function(e) {
     if (!e.isNormalClick()) {
@@ -13,23 +14,64 @@ JX.behavior('harbormaster-log', function(config) {
 
     e.kill();
 
-    var row = e.getNode('tag:tr');
+    expand(e.getTarget());
+  });
+
+  function expand(node) {
+    var row = JX.DOM.findAbove(node, 'tr');
     row = JX.DOM.findAbove(row, 'tr');
 
-    var data = e.getNodeData('harbormaster-log-expand');
+    var data = JX.Stratcom.getData(node);
 
     var uri = new JX.URI(config.renderURI)
       .addQueryParams(data);
 
+    if (data.live) {
+      following = true;
+    }
+
     var request = new JX.Request(uri, function(r) {
       var result = JX.$H(r.markup).getNode();
       var rows = [].slice.apply(result.firstChild.childNodes);
 
+      // If we're following the bottom of the log, the result always includes
+      // the last line from the previous render. Throw it away, then add the
+      // new data.
+      if (data.live && row.previousSibling) {
+        JX.DOM.remove(row.previousSibling);
+      }
+
       JX.DOM.replace(row, rows);
+
+      if (data.live) {
+        // If this was a live follow, scroll the new data into view. This is
+        // probably intensely annoying in practice but seems cool for now.
+        var last_row = rows[rows.length - 1];
+        var tail_pos = JX.$V(last_row).y + JX.Vector.getDim(last_row).y;
+        var view_y = JX.Vector.getViewport().y;
+        JX.DOM.scrollToPosition(null, (tail_pos - view_y) + 32);
+
+        setTimeout(follow, 500);
+      }
     });
 
     request.send();
-  });
+  }
+
+  function follow() {
+    if (!following) {
+      return;
+    }
+
+    var live;
+    try {
+      live = JX.DOM.find(contentNode, 'a', 'harbormaster-log-live');
+    } catch (e) {
+      return;
+    }
+
+    expand(live);
+  }
 
   function onresponse(r) {
     JX.DOM.alterClass(contentNode, 'harbormaster-log-view-loading', false);

From a5efd7eedb3cd3245928e214b0b8123be13427c7 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Wed, 28 Feb 2018 14:15:02 -0800
Subject: [PATCH 25/39] Add "object-src 'none'" to the Content-Security-Policy

Summary: See PHI399. Ref T4340. We don't require Flash/Java anywhere and can safely block them unconditionally in the Content-Security-Policy header.

Test Plan: Added a `<object ... />` tag to a page, saw "Blocked Plug-In" and a CSP warning in the browser console.

Maniphest Tasks: T4340

Differential Revision: https://secure.phabricator.com/D19154
---
 src/aphront/response/AphrontResponse.php | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/aphront/response/AphrontResponse.php b/src/aphront/response/AphrontResponse.php
index 34cbd4081f..56d074bd6f 100644
--- a/src/aphront/response/AphrontResponse.php
+++ b/src/aphront/response/AphrontResponse.php
@@ -144,6 +144,9 @@ abstract class AphrontResponse extends Phobject {
       $csp[] = "frame-ancestors 'none'";
     }
 
+    // Block relics of the old world: Flash, Java applets, and so on.
+    $csp[] = "object-src 'none'";
+
     $csp = implode('; ', $csp);
 
     return $csp;

From afc98f5d5d0808c8ad2109fb9dc0901e65eb3bda Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Wed, 28 Feb 2018 15:19:46 -0800
Subject: [PATCH 26/39] Remove defunct "download" route in Files pointing to
 nonexistent controller

Summary:
Depends on D19154. Ref T13094. This controller was removed at some point and this route no longer works.

I plan to add a new `download/` route to let us tighten the `form-action` Content Security Policy.

Test Plan: Grepped for the route and controller, no hits.

Maniphest Tasks: T13094

Differential Revision: https://secure.phabricator.com/D19155
---
 .../files/application/PhabricatorFilesApplication.php            | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/applications/files/application/PhabricatorFilesApplication.php b/src/applications/files/application/PhabricatorFilesApplication.php
index 1010818c19..c3903ffdce 100644
--- a/src/applications/files/application/PhabricatorFilesApplication.php
+++ b/src/applications/files/application/PhabricatorFilesApplication.php
@@ -86,7 +86,6 @@ final class PhabricatorFilesApplication extends PhabricatorApplication {
           'PhabricatorFileTransformListController',
         'uploaddialog/(?P<single>single/)?'
           => 'PhabricatorFileUploadDialogController',
-        'download/(?P<phid>[^/]+)/' => 'PhabricatorFileDialogController',
         'iconset/(?P<key>[^/]+)/' => array(
           'select/' => 'PhabricatorFileIconSetSelectController',
         ),

From ab579f2511102775d27bda92c6f3d3c8fc302d3c Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Wed, 28 Feb 2018 15:22:10 -0800
Subject: [PATCH 27/39] Never generate file download forms which point to the
 CDN domain, tighten "form-action" CSP

Summary:
Depends on D19155. Ref T13094. Ref T4340.

We can't currently implement a strict `form-action 'self'` content security policy because some file downloads rely on a `<form />` which sometimes POSTs to the CDN domain.

Broadly, stop generating these forms. We just redirect instead, and show an interstitial confirm dialog if no CDN domain is configured. This makes the UX for installs with no CDN domain a little worse and the UX for everyone else better.

Then, implement the stricter Content-Security-Policy.

This also removes extra confirm dialogs for downloading Harbormaster build logs and data exports.

Test Plan:
  - Went through the plain data export, data export with bulk jobs, ssh key generation, calendar ICS download, Diffusion data, Paste data, Harbormaster log data, and normal file data download workflows with a CDN domain.
  - Went through all those workflows again without a CDN domain.
  - Grepped for affected symbols (`getCDNURI()`, `getDownloadURI()`).
  - Added an evil form to a page, tried to submit it, was rejected.
  - Went through the ReCaptcha and Stripe flows again to see if they're submitting any forms.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13094, T4340

Differential Revision: https://secure.phabricator.com/D19156
---
 resources/celerity/map.php                    | 28 ++++++-------
 .../response/AphrontRedirectResponse.php      | 10 +++++
 src/aphront/response/AphrontResponse.php      |  7 ++++
 ...habricatorAuthSSHKeyGenerateController.php | 30 +++++++++-----
 .../base/controller/PhabricatorController.php |  1 +
 .../controller/DiffusionServeController.php   |  2 +-
 .../PhabricatorFilesApplication.php           |  4 +-
 .../PhabricatorFileDataController.php         | 39 +++++++++----------
 .../PhabricatorFileInfoController.php         |  3 +-
 .../PhabricatorEmbedFileRemarkupRule.php      |  2 +-
 .../files/storage/PhabricatorFile.php         | 28 ++++++++++---
 ...HarbormasterBuildLogDownloadController.php | 14 +------
 .../view/HarbormasterBuildLogView.php         |  6 ++-
 ...PhabricatorApplicationSearchController.php | 18 ++++-----
 .../PhabricatorExportEngineBulkJobType.php    |  1 -
 .../rsrc/externals/javelin/lib/Workflow.js    |  7 ++++
 16 files changed, 118 insertions(+), 82 deletions(-)

diff --git a/resources/celerity/map.php b/resources/celerity/map.php
index a59db7ed05..42a4c854ef 100644
--- a/resources/celerity/map.php
+++ b/resources/celerity/map.php
@@ -10,7 +10,7 @@ return array(
     'conpherence.pkg.css' => 'e68cf1fa',
     'conpherence.pkg.js' => '15191c65',
     'core.pkg.css' => '2fa91e14',
-    'core.pkg.js' => '7aa5bd92',
+    'core.pkg.js' => 'e7ce7bba',
     'darkconsole.pkg.js' => '1f9a31bc',
     'differential.pkg.css' => '113e692c',
     'differential.pkg.js' => 'f6d809c0',
@@ -255,7 +255,7 @@ return array(
     'rsrc/externals/javelin/lib/URI.js' => 'c989ade3',
     'rsrc/externals/javelin/lib/Vector.js' => '2caa8fb8',
     'rsrc/externals/javelin/lib/WebSocket.js' => '3ffe32d6',
-    'rsrc/externals/javelin/lib/Workflow.js' => '1e911d0f',
+    'rsrc/externals/javelin/lib/Workflow.js' => '0eb1db0c',
     'rsrc/externals/javelin/lib/__tests__/Cookie.js' => '5ed109e8',
     'rsrc/externals/javelin/lib/__tests__/DOM.js' => 'c984504b',
     'rsrc/externals/javelin/lib/__tests__/JSON.js' => '837a7d68',
@@ -757,7 +757,7 @@ return array(
     'javelin-workboard-card' => 'c587b80f',
     'javelin-workboard-column' => '758b4758',
     'javelin-workboard-controller' => '26167537',
-    'javelin-workflow' => '1e911d0f',
+    'javelin-workflow' => '0eb1db0c',
     'maniphest-report-css' => '9b9580b7',
     'maniphest-task-edit-css' => 'fda62a9b',
     'maniphest-task-summary-css' => '11cc5344',
@@ -977,6 +977,17 @@ return array(
       'javelin-dom',
       'javelin-router',
     ),
+    '0eb1db0c' => array(
+      'javelin-stratcom',
+      'javelin-request',
+      'javelin-dom',
+      'javelin-vector',
+      'javelin-install',
+      'javelin-util',
+      'javelin-mask',
+      'javelin-uri',
+      'javelin-routable',
+    ),
     '0f764c35' => array(
       'javelin-install',
       'javelin-util',
@@ -1035,17 +1046,6 @@ return array(
       'javelin-request',
       'javelin-uri',
     ),
-    '1e911d0f' => array(
-      'javelin-stratcom',
-      'javelin-request',
-      'javelin-dom',
-      'javelin-vector',
-      'javelin-install',
-      'javelin-util',
-      'javelin-mask',
-      'javelin-uri',
-      'javelin-routable',
-    ),
     '1f6794f6' => array(
       'javelin-behavior',
       'javelin-stratcom',
diff --git a/src/aphront/response/AphrontRedirectResponse.php b/src/aphront/response/AphrontRedirectResponse.php
index 5b4b009796..390ad193c9 100644
--- a/src/aphront/response/AphrontRedirectResponse.php
+++ b/src/aphront/response/AphrontRedirectResponse.php
@@ -8,6 +8,7 @@ class AphrontRedirectResponse extends AphrontResponse {
   private $uri;
   private $stackWhenCreated;
   private $isExternal;
+  private $closeDialogBeforeRedirect;
 
   public function setIsExternal($external) {
     $this->isExternal = $external;
@@ -37,6 +38,15 @@ class AphrontRedirectResponse extends AphrontResponse {
     return PhabricatorEnv::getEnvConfig('debug.stop-on-redirect');
   }
 
+  public function setCloseDialogBeforeRedirect($close) {
+    $this->closeDialogBeforeRedirect = $close;
+    return $this;
+  }
+
+  public function getCloseDialogBeforeRedirect() {
+    return $this->closeDialogBeforeRedirect;
+  }
+
   public function getHeaders() {
     $headers = array();
     if (!$this->shouldStopForDebugging()) {
diff --git a/src/aphront/response/AphrontResponse.php b/src/aphront/response/AphrontResponse.php
index 56d074bd6f..ed3d6d3435 100644
--- a/src/aphront/response/AphrontResponse.php
+++ b/src/aphront/response/AphrontResponse.php
@@ -147,6 +147,13 @@ abstract class AphrontResponse extends Phobject {
     // Block relics of the old world: Flash, Java applets, and so on.
     $csp[] = "object-src 'none'";
 
+    // Don't allow forms to submit offsite.
+
+    // This can result in some trickiness with file downloads if applications
+    // try to start downloads by submitting a dialog. Redirect to the file's
+    // download URI instead of submitting a form to it.
+    $csp[] = "form-action 'self'";
+
     $csp = implode('; ', $csp);
 
     return $csp;
diff --git a/src/applications/auth/controller/PhabricatorAuthSSHKeyGenerateController.php b/src/applications/auth/controller/PhabricatorAuthSSHKeyGenerateController.php
index f39707618d..206a2f1c4a 100644
--- a/src/applications/auth/controller/PhabricatorAuthSSHKeyGenerateController.php
+++ b/src/applications/auth/controller/PhabricatorAuthSSHKeyGenerateController.php
@@ -24,10 +24,12 @@ final class PhabricatorAuthSSHKeyGenerateController
       $keys = PhabricatorSSHKeyGenerator::generateKeypair();
       list($public_key, $private_key) = $keys;
 
+      $key_name = $default_name.'.key';
+
       $file = PhabricatorFile::newFromFileData(
         $private_key,
         array(
-          'name' => $default_name.'.key',
+          'name' => $key_name,
           'ttl.relative' => phutil_units('10 minutes in seconds'),
           'viewPolicy' => $viewer->getPHID(),
         ));
@@ -62,25 +64,33 @@ final class PhabricatorAuthSSHKeyGenerateController
         ->setContentSourceFromRequest($request)
         ->applyTransactions($key, $xactions);
 
-      // NOTE: We're disabling workflow on submit so the download works. We're
-      // disabling workflow on cancel so the page reloads, showing the new
-      // key.
+      $download_link = phutil_tag(
+        'a',
+        array(
+          'href' => $file->getDownloadURI(),
+        ),
+        array(
+          id(new PHUIIconView())->setIcon('fa-download'),
+          ' ',
+          pht('Download Private Key (%s)', $key_name),
+        ));
+      $download_link = phutil_tag('strong', array(), $download_link);
+
+      // NOTE: We're disabling workflow on cancel so the page reloads, showing
+      // the new key.
 
       return $this->newDialog()
         ->setTitle(pht('Download Private Key'))
-        ->setDisableWorkflowOnCancel(true)
-        ->setDisableWorkflowOnSubmit(true)
-        ->setSubmitURI($file->getDownloadURI())
         ->appendParagraph(
           pht(
             'A keypair has been generated, and the public key has been '.
-            'added as a recognized key. Use the button below to download '.
-            'the private key.'))
+            'added as a recognized key.'))
+        ->appendParagraph($download_link)
         ->appendParagraph(
           pht(
             'After you download the private key, it will be destroyed. '.
             'You will not be able to retrieve it if you lose your copy.'))
-        ->addSubmitButton(pht('Download Private Key'))
+        ->setDisableWorkflowOnCancel(true)
         ->addCancelButton($cancel_uri, pht('Done'));
     }
 
diff --git a/src/applications/base/controller/PhabricatorController.php b/src/applications/base/controller/PhabricatorController.php
index 5952a806ed..df0c94c13d 100644
--- a/src/applications/base/controller/PhabricatorController.php
+++ b/src/applications/base/controller/PhabricatorController.php
@@ -298,6 +298,7 @@ abstract class PhabricatorController extends AphrontController {
           ->setContent(
             array(
               'redirect' => $response->getURI(),
+              'close' => $response->getCloseDialogBeforeRedirect(),
             ));
       }
     }
diff --git a/src/applications/diffusion/controller/DiffusionServeController.php b/src/applications/diffusion/controller/DiffusionServeController.php
index c0f72ae404..ac0b993b2f 100644
--- a/src/applications/diffusion/controller/DiffusionServeController.php
+++ b/src/applications/diffusion/controller/DiffusionServeController.php
@@ -1046,7 +1046,7 @@ final class DiffusionServeController extends DiffusionController {
           // <https://github.com/github/git-lfs/issues/1088>
           $no_authorization = 'Basic '.base64_encode('none');
 
-          $get_uri = $file->getCDNURI();
+          $get_uri = $file->getCDNURI('data');
           $actions['download'] = array(
             'href' => $get_uri,
             'header' => array(
diff --git a/src/applications/files/application/PhabricatorFilesApplication.php b/src/applications/files/application/PhabricatorFilesApplication.php
index c3903ffdce..2d3a1b35c6 100644
--- a/src/applications/files/application/PhabricatorFilesApplication.php
+++ b/src/applications/files/application/PhabricatorFilesApplication.php
@@ -101,7 +101,7 @@ final class PhabricatorFilesApplication extends PhabricatorApplication {
 
   private function getResourceSubroutes() {
     return array(
-      'data/'.
+      '(?P<kind>data|download)/'.
         '(?:@(?P<instance>[^/]+)/)?'.
         '(?P<key>[^/]+)/'.
         '(?P<phid>[^/]+)/'.
@@ -132,7 +132,7 @@ final class PhabricatorFilesApplication extends PhabricatorApplication {
 
   public function getQuicksandURIPatternBlacklist() {
     return array(
-      '/file/data/.*',
+      '/file/(data|download)/.*',
     );
   }
 
diff --git a/src/applications/files/controller/PhabricatorFileDataController.php b/src/applications/files/controller/PhabricatorFileDataController.php
index 98f2cdbd51..91dfb4fa4f 100644
--- a/src/applications/files/controller/PhabricatorFileDataController.php
+++ b/src/applications/files/controller/PhabricatorFileDataController.php
@@ -26,6 +26,9 @@ final class PhabricatorFileDataController extends PhabricatorFileController {
     $req_domain = $request->getHost();
     $main_domain = id(new PhutilURI($base_uri))->getDomain();
 
+    $request_kind = $request->getURIData('kind');
+    $is_download = ($request_kind === 'download');
+
     if (!strlen($alt) || $main_domain == $alt_domain) {
       // No alternate domain.
       $should_redirect = false;
@@ -50,7 +53,7 @@ final class PhabricatorFileDataController extends PhabricatorFileController {
     if ($should_redirect) {
       return id(new AphrontRedirectResponse())
         ->setIsExternal(true)
-        ->setURI($file->getCDNURI());
+        ->setURI($file->getCDNURI($request_kind));
     }
 
     $response = new AphrontFileResponse();
@@ -71,34 +74,30 @@ final class PhabricatorFileDataController extends PhabricatorFileController {
     }
 
     $is_viewable = $file->isViewableInBrowser();
-    $force_download = $request->getExists('download');
-
     $request_type = $request->getHTTPHeader('X-Phabricator-Request-Type');
     $is_lfs = ($request_type == 'git-lfs');
 
-    if ($is_viewable && !$force_download) {
+    if ($is_viewable && !$is_download) {
       $response->setMimeType($file->getViewableMimeType());
     } else {
-      $is_public = !$viewer->isLoggedIn();
       $is_post = $request->isHTTPPost();
 
-      // NOTE: Require POST to download files from the primary domain if the
-      // request includes credentials. The "Download File" links we generate
-      // in the web UI are forms which use POST to satisfy this requirement.
+      // NOTE: Require POST to download files from the primary domain. If the
+      // request is not a POST request but arrives on the primary domain, we
+      // render a confirmation dialog. For discussion, see T13094.
 
-      // The intent is to make attacks based on tags like "<iframe />" and
-      // "<script />" (which can issue GET requests, but can not easily issue
-      // POST requests) more difficult to execute.
-
-      // The best defense against these attacks is to use an alternate file
-      // domain, which is why we strongly recommend doing so.
-
-      $is_safe = ($is_alternate_domain || $is_lfs || $is_post || $is_public);
+      $is_safe = ($is_alternate_domain || $is_lfs || $is_post);
       if (!$is_safe) {
-        // This is marked as "external" because it is fully qualified.
-        return id(new AphrontRedirectResponse())
-          ->setIsExternal(true)
-          ->setURI(PhabricatorEnv::getProductionURI($file->getBestURI()));
+        return $this->newDialog()
+          ->setSubmitURI($file->getDownloadURI())
+          ->setTitle(pht('Download File'))
+          ->appendParagraph(
+            pht(
+              'Download file %s (%s)?',
+              phutil_tag('strong', array(), $file->getName()),
+              phutil_format_bytes($file->getByteSize())))
+          ->addCancelButton($file->getURI())
+          ->addSubmitButton(pht('Download File'));
       }
 
       $response->setMimeType($file->getMimeType());
diff --git a/src/applications/files/controller/PhabricatorFileInfoController.php b/src/applications/files/controller/PhabricatorFileInfoController.php
index f30421c132..976324d0b2 100644
--- a/src/applications/files/controller/PhabricatorFileInfoController.php
+++ b/src/applications/files/controller/PhabricatorFileInfoController.php
@@ -137,11 +137,10 @@ final class PhabricatorFileInfoController extends PhabricatorFileController {
       $curtain->addAction(
         id(new PhabricatorActionView())
           ->setUser($viewer)
-          ->setRenderAsForm($can_download)
           ->setDownload($can_download)
           ->setName(pht('Download File'))
           ->setIcon('fa-download')
-          ->setHref($file->getViewURI())
+          ->setHref($file->getDownloadURI())
           ->setDisabled(!$can_download)
           ->setWorkflow(!$can_download));
     }
diff --git a/src/applications/files/markup/PhabricatorEmbedFileRemarkupRule.php b/src/applications/files/markup/PhabricatorEmbedFileRemarkupRule.php
index e4fe826f5b..9cbc119c84 100644
--- a/src/applications/files/markup/PhabricatorEmbedFileRemarkupRule.php
+++ b/src/applications/files/markup/PhabricatorEmbedFileRemarkupRule.php
@@ -144,7 +144,7 @@ final class PhabricatorEmbedFileRemarkupRule
 
           $existing_xform = $file->getTransform($preview_key);
           if ($existing_xform) {
-            $xform_uri = $existing_xform->getCDNURI();
+            $xform_uri = $existing_xform->getCDNURI('data');
           } else {
             $xform_uri = $file->getURIForTransform($xform);
           }
diff --git a/src/applications/files/storage/PhabricatorFile.php b/src/applications/files/storage/PhabricatorFile.php
index 2b8624b9c0..29fb606a4b 100644
--- a/src/applications/files/storage/PhabricatorFile.php
+++ b/src/applications/files/storage/PhabricatorFile.php
@@ -810,16 +810,24 @@ final class PhabricatorFile extends PhabricatorFileDAO
         pht('You must save a file before you can generate a view URI.'));
     }
 
-    return $this->getCDNURI();
+    return $this->getCDNURI('data');
   }
 
-  public function getCDNURI() {
+  public function getCDNURI($request_kind) {
+    if (($request_kind !== 'data') &&
+        ($request_kind !== 'download')) {
+      throw new Exception(
+        pht(
+          'Unknown file content request kind "%s".',
+          $request_kind));
+    }
+
     $name = self::normalizeFileName($this->getName());
     $name = phutil_escape_uri($name);
 
     $parts = array();
     $parts[] = 'file';
-    $parts[] = 'data';
+    $parts[] = $request_kind;
 
     // If this is an instanced install, add the instance identifier to the URI.
     // Instanced configurations behind a CDN may not be able to control the
@@ -861,9 +869,7 @@ final class PhabricatorFile extends PhabricatorFileDAO
   }
 
   public function getDownloadURI() {
-    $uri = id(new PhutilURI($this->getViewURI()))
-      ->setQueryParam('download', true);
-    return (string)$uri;
+    return $this->getCDNURI('download');
   }
 
   public function getURIForTransform(PhabricatorFileTransform $transform) {
@@ -1469,6 +1475,16 @@ final class PhabricatorFile extends PhabricatorFileDAO
       ->setURI($uri);
   }
 
+  public function newDownloadResponse() {
+    // We're cheating a little bit here and relying on the fact that
+    // getDownloadURI() always returns a fully qualified URI with a complete
+    // domain.
+    return id(new AphrontRedirectResponse())
+      ->setIsExternal(true)
+      ->setCloseDialogBeforeRedirect(true)
+      ->setURI($this->getDownloadURI());
+  }
+
   public function attachTransforms(array $map) {
     $this->transforms = $map;
     return $this;
diff --git a/src/applications/harbormaster/controller/HarbormasterBuildLogDownloadController.php b/src/applications/harbormaster/controller/HarbormasterBuildLogDownloadController.php
index 6bdd04edc3..40313cdafd 100644
--- a/src/applications/harbormaster/controller/HarbormasterBuildLogDownloadController.php
+++ b/src/applications/harbormaster/controller/HarbormasterBuildLogDownloadController.php
@@ -44,19 +44,7 @@ final class HarbormasterBuildLogDownloadController
         ->addCancelButton($cancel_uri);
     }
 
-    $size = $file->getByteSize();
-
-    return $this->newDialog()
-      ->setTitle(pht('Download Build Log'))
-      ->appendParagraph(
-        pht(
-          'This log has a total size of %s. If you insist, you may '.
-          'download it.',
-          phutil_tag('strong', array(), phutil_format_bytes($size))))
-      ->setDisableWorkflowOnSubmit(true)
-      ->addSubmitButton(pht('Download Log'))
-      ->setSubmitURI($file->getDownloadURI())
-      ->addCancelButton($cancel_uri, pht('Done'));
+    return $file->newDownloadResponse();
   }
 
 }
diff --git a/src/applications/harbormaster/view/HarbormasterBuildLogView.php b/src/applications/harbormaster/view/HarbormasterBuildLogView.php
index e240aab510..7200cdce70 100644
--- a/src/applications/harbormaster/view/HarbormasterBuildLogView.php
+++ b/src/applications/harbormaster/view/HarbormasterBuildLogView.php
@@ -34,12 +34,14 @@ final class HarbormasterBuildLogView extends AphrontView {
 
     $download_uri = "/harbormaster/log/download/{$id}/";
 
+    $can_download = (bool)$log->getFilePHID();
+
     $download_button = id(new PHUIButtonView())
       ->setTag('a')
       ->setHref($download_uri)
       ->setIcon('fa-download')
-      ->setDisabled(!$log->getFilePHID())
-      ->setWorkflow(true)
+      ->setDisabled(!$can_download)
+      ->setWorkflow(!$can_download)
       ->setText(pht('Download Log'));
 
     $header->addActionLink($download_button);
diff --git a/src/applications/search/controller/PhabricatorApplicationSearchController.php b/src/applications/search/controller/PhabricatorApplicationSearchController.php
index cc8bcefff1..cf4f95c16d 100644
--- a/src/applications/search/controller/PhabricatorApplicationSearchController.php
+++ b/src/applications/search/controller/PhabricatorApplicationSearchController.php
@@ -512,15 +512,7 @@ final class PhabricatorApplicationSearchController
             ->setURI($job->getMonitorURI());
         } else {
           $file = $export_engine->exportFile();
-
-          return $this->newDialog()
-            ->setTitle(pht('Download Results'))
-            ->appendParagraph(
-              pht('Click the download button to download the exported data.'))
-            ->addCancelButton($cancel_uri, pht('Done'))
-            ->setSubmitURI($file->getDownloadURI())
-            ->setDisableWorkflowOnSubmit(true)
-            ->addSubmitButton(pht('Download Data'));
+          return $file->newDownloadResponse();
         }
       }
     }
@@ -535,12 +527,18 @@ final class PhabricatorApplicationSearchController
           ->setValue($format_key)
           ->setOptions($format_options));
 
+    if ($is_large_export) {
+      $submit_button = pht('Continue');
+    } else {
+      $submit_button = pht('Download Data');
+    }
+
     return $this->newDialog()
       ->setTitle(pht('Export Results'))
       ->setErrors($errors)
       ->appendForm($export_form)
       ->addCancelButton($cancel_uri)
-      ->addSubmitButton(pht('Continue'));
+      ->addSubmitButton($submit_button);
   }
 
   private function processEditRequest() {
diff --git a/src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php b/src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php
index 712127f479..cf54bc7995 100644
--- a/src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php
+++ b/src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php
@@ -36,7 +36,6 @@ final class PhabricatorExportEngineBulkJobType
           ->setName(pht('Temporary File Expired'));
       } else {
         $actions[] = id(new PhabricatorActionView())
-          ->setRenderAsForm(true)
           ->setHref($file->getDownloadURI())
           ->setIcon('fa-download')
           ->setName(pht('Download Data Export'));
diff --git a/webroot/rsrc/externals/javelin/lib/Workflow.js b/webroot/rsrc/externals/javelin/lib/Workflow.js
index 7f0818d4e0..995f204c5d 100644
--- a/webroot/rsrc/externals/javelin/lib/Workflow.js
+++ b/webroot/rsrc/externals/javelin/lib/Workflow.js
@@ -276,6 +276,13 @@ JX.install('Workflow', {
       // It is permissible to send back a falsey redirect to force a page
       // reload, so we need to take this branch if the key is present.
       if (r && (typeof r.redirect != 'undefined')) {
+        // Before we redirect to file downloads, we close the dialog. These
+        // redirects aren't real navigation events so we end up stuck in the
+        // dialog otherwise.
+        if (r.close) {
+          this._pop();
+        }
+
         JX.$U(r.redirect).go();
       } else if (r && r.dialog) {
         this._push();

From a2fdf14275f932fcf70baf345ec2d944ffbd5a45 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Wed, 28 Feb 2018 17:04:40 -0800
Subject: [PATCH 28/39] Stop using forms to download files in file embed and
 lightbox elements

Summary: Depends on D19156. Ref T13094. This replaces the remaining forms in the file embed view and lightbox with normal download links.

Test Plan: Clicked "Download" and lightbox -> download for embedded files.

Maniphest Tasks: T13094

Differential Revision: https://secure.phabricator.com/D19157
---
 resources/celerity/map.php                    | 24 ++++----
 src/view/layout/PhabricatorFileLinkView.php   | 20 ++-----
 src/view/page/PhabricatorStandardPageView.php | 30 +---------
 .../js/core/behavior-lightbox-attachments.js  | 59 +++++++++----------
 4 files changed, 47 insertions(+), 86 deletions(-)

diff --git a/resources/celerity/map.php b/resources/celerity/map.php
index 42a4c854ef..4eebc07609 100644
--- a/resources/celerity/map.php
+++ b/resources/celerity/map.php
@@ -10,7 +10,7 @@ return array(
     'conpherence.pkg.css' => 'e68cf1fa',
     'conpherence.pkg.js' => '15191c65',
     'core.pkg.css' => '2fa91e14',
-    'core.pkg.js' => 'e7ce7bba',
+    'core.pkg.js' => 'a3ceffdb',
     'darkconsole.pkg.js' => '1f9a31bc',
     'differential.pkg.css' => '113e692c',
     'differential.pkg.js' => 'f6d809c0',
@@ -494,7 +494,7 @@ return array(
     'rsrc/js/core/behavior-hovercard.js' => 'bcaccd64',
     'rsrc/js/core/behavior-keyboard-pager.js' => 'a8da01f0',
     'rsrc/js/core/behavior-keyboard-shortcuts.js' => '01fca1f0',
-    'rsrc/js/core/behavior-lightbox-attachments.js' => '560f41da',
+    'rsrc/js/core/behavior-lightbox-attachments.js' => 'e31fad01',
     'rsrc/js/core/behavior-line-linker.js' => '1499a8cb',
     'rsrc/js/core/behavior-more.js' => 'a80d0378',
     'rsrc/js/core/behavior-object-selector.js' => '77c1f0b0',
@@ -642,7 +642,7 @@ return array(
     'javelin-behavior-history-install' => '7ee2b591',
     'javelin-behavior-icon-composer' => '8499b6ab',
     'javelin-behavior-launch-icon-composer' => '48086888',
-    'javelin-behavior-lightbox-attachments' => '560f41da',
+    'javelin-behavior-lightbox-attachments' => 'e31fad01',
     'javelin-behavior-line-chart' => 'e4232876',
     'javelin-behavior-load-blame' => '42126667',
     'javelin-behavior-maniphest-batch-selector' => 'ad54037e',
@@ -1359,15 +1359,6 @@ return array(
       'javelin-vector',
       'javelin-dom',
     ),
-    '560f41da' => array(
-      'javelin-behavior',
-      'javelin-stratcom',
-      'javelin-dom',
-      'javelin-mask',
-      'javelin-util',
-      'phuix-icon-view',
-      'phabricator-busy',
-    ),
     '58dea2fa' => array(
       'javelin-install',
       'javelin-util',
@@ -2070,6 +2061,15 @@ return array(
       'javelin-dom',
       'phabricator-draggable-list',
     ),
+    'e31fad01' => array(
+      'javelin-behavior',
+      'javelin-stratcom',
+      'javelin-dom',
+      'javelin-mask',
+      'javelin-util',
+      'phuix-icon-view',
+      'phabricator-busy',
+    ),
     'e379b58e' => array(
       'javelin-behavior',
       'javelin-stratcom',
diff --git a/src/view/layout/PhabricatorFileLinkView.php b/src/view/layout/PhabricatorFileLinkView.php
index b0649381dd..82715c0294 100644
--- a/src/view/layout/PhabricatorFileLinkView.php
+++ b/src/view/layout/PhabricatorFileLinkView.php
@@ -134,21 +134,13 @@ final class PhabricatorFileLinkView extends AphrontTagView {
     $dl_icon = id(new PHUIIconView())
       ->setIcon('fa-download');
 
-    $download_form = phabricator_form(
-      $this->getViewer(),
+    $download_link = phutil_tag(
+      'a',
       array(
-        'action' => $this->getFileDownloadURI(),
-        'method' => 'POST',
-        'class'  => 'embed-download-form',
-        'sigil'  => 'embed-download-form download',
+        'class' => 'phabricator-remarkup-embed-layout-download',
+        'href' => $this->getFileDownloadURI(),
       ),
-      phutil_tag(
-        'button',
-        array(
-          'class' => 'phabricator-remarkup-embed-layout-download',
-          'type' => 'submit',
-        ),
-        pht('Download')));
+      pht('Download'));
 
     $info = phutil_tag(
       'span',
@@ -177,7 +169,7 @@ final class PhabricatorFileLinkView extends AphrontTagView {
     return array(
       $icon,
       $inner,
-      $download_form,
+      $download_link,
     );
   }
 }
diff --git a/src/view/page/PhabricatorStandardPageView.php b/src/view/page/PhabricatorStandardPageView.php
index 266e6592ee..8ee7566969 100644
--- a/src/view/page/PhabricatorStandardPageView.php
+++ b/src/view/page/PhabricatorStandardPageView.php
@@ -279,35 +279,7 @@ final class PhabricatorStandardPageView extends PhabricatorBarePageView
         }
       }
 
-      $icon = id(new PHUIIconView())
-        ->setIcon('fa-download')
-        ->addClass('phui-icon-circle-icon');
-      $lightbox_id = celerity_generate_unique_node_id();
-      $download_form = phabricator_form(
-        $user,
-        array(
-          'action' => '#',
-          'method' => 'POST',
-          'class'  => 'lightbox-download-form',
-          'sigil'  => 'download lightbox-download-submit',
-          'id'     => 'lightbox-download-form',
-        ),
-        phutil_tag(
-          'a',
-          array(
-            'class' => 'lightbox-download phui-icon-circle hover-green',
-            'href' => '#',
-          ),
-          array(
-            $icon,
-          )));
-
-      Javelin::initBehavior(
-        'lightbox-attachments',
-        array(
-          'lightbox_id'     => $lightbox_id,
-          'downloadForm'    => $download_form,
-        ));
+      Javelin::initBehavior('lightbox-attachments');
     }
 
     Javelin::initBehavior('aphront-form-disable-on-submit');
diff --git a/webroot/rsrc/js/core/behavior-lightbox-attachments.js b/webroot/rsrc/js/core/behavior-lightbox-attachments.js
index b9ea0dd8db..eb31d22c60 100644
--- a/webroot/rsrc/js/core/behavior-lightbox-attachments.js
+++ b/webroot/rsrc/js/core/behavior-lightbox-attachments.js
@@ -9,19 +9,18 @@
  *           phabricator-busy
  */
 
-JX.behavior('lightbox-attachments', function (config) {
+JX.behavior('lightbox-attachments', function() {
+
+  var lightbox = null;
 
-  var lightbox     = null;
   var prev         = null;
   var next         = null;
   var shown        = false;
-  var downloadForm = JX.$H(config.downloadForm).getFragment().firstChild;
-  var lightbox_id  = config.lightbox_id;
 
   function _toggleComment(e) {
     e.kill();
     shown = !shown;
-    JX.DOM.alterClass(JX.$(lightbox_id), 'comment-panel-open', shown);
+    JX.DOM.alterClass(lightbox, 'comment-panel-open', shown);
   }
 
   function markCommentsLoading(loading) {
@@ -48,6 +47,12 @@ JX.behavior('lightbox-attachments', function (config) {
       return;
     }
 
+    // If you click the "Download" link inside an embedded file element,
+    // don't lightbox the file.
+    if (e.getNode('tag:a')) {
+      return;
+    }
+
     e.kill();
 
     var mainFrame = JX.$('main-page-frame');
@@ -82,8 +87,7 @@ JX.behavior('lightbox-attachments', function (config) {
     var img_uri = '';
     var img = '';
     var extra_status = '';
-    // for now, this conditional is always true
-    // revisit if / when we decide to add non-images to lightbox view
+
     if (target_data.viewable) {
       img_uri = target_data.uri;
       var alt_name = '';
@@ -114,7 +118,7 @@ JX.behavior('lightbox-attachments', function (config) {
           {
             className : 'lightbox-icon-frame',
             sigil : 'lightbox-download-submit',
-            href : '#',
+            href : target_data.dUri,
           },
           [ imgIcon, nameElement ]
         );
@@ -138,12 +142,12 @@ JX.behavior('lightbox-attachments', function (config) {
       );
 
     var commentClass = (shown) ? 'comment-panel-open' : '';
+
     lightbox =
       JX.$N('div',
         {
           className : 'lightbox-attachment ' + commentClass,
-          sigil : 'lightbox-attachment',
-          id : lightbox_id
+          sigil : 'lightbox-attachment'
         },
         [imgFrame, commentFrame]
       );
@@ -161,12 +165,17 @@ JX.behavior('lightbox-attachments', function (config) {
         ]
       );
 
-    var downloadSpan =
-      JX.$N('span',
-        {
-          className : 'lightbox-download'
-        }
-      );
+    var download_icon = new JX.PHUIXIconView()
+      .setIcon('fa-download phui-icon-circle-icon')
+      .getNode();
+
+    var download_button = JX.$N(
+      'a',
+      {
+        className: 'lightbox-download phui-icon-circle hover-sky',
+        href: target_data.dUri
+      },
+      download_icon);
 
     var commentIcon = new JX.PHUIXIconView()
       .setIcon('fa-comments phui-icon-circle-icon')
@@ -180,6 +189,7 @@ JX.behavior('lightbox-attachments', function (config) {
         },
         commentIcon
       );
+
     var closeIcon = new JX.PHUIXIconView()
       .setIcon('fa-times phui-icon-circle-icon')
       .getNode();
@@ -190,12 +200,13 @@ JX.behavior('lightbox-attachments', function (config) {
           href : '#'
         },
         closeIcon);
+
     var statusHTML =
       JX.$N('div',
         {
           className : 'lightbox-status'
         },
-       [statusSpan, closeButton, commentButton, downloadSpan]
+       [statusSpan, closeButton, commentButton, download_button]
       );
     JX.DOM.appendContent(lightbox, statusHTML);
     JX.DOM.listen(closeButton, 'click', null, closeLightBox);
@@ -246,9 +257,6 @@ JX.behavior('lightbox-attachments', function (config) {
     JX.DOM.alterClass(document.body, 'lightbox-attached', true);
     JX.Mask.show('jx-dark-mask');
 
-    downloadForm.action = target_data.dUri;
-    downloadSpan.appendChild(downloadForm);
-
     document.body.appendChild(lightbox);
 
     if (img_uri) {
@@ -365,23 +373,12 @@ JX.behavior('lightbox-attachments', function (config) {
     'lightbox-comment-form',
     _sendMessage);
 
-  var _startDownload = function(e) {
-    e.kill();
-    var form = JX.$('lightbox-download-form');
-    form.submit();
-  };
-
   var _startPageDownload = function(e) {
     e.kill();
     var form = e.getNode('tag:form');
     form.submit();
   };
 
-  JX.Stratcom.listen(
-    'click',
-    'lightbox-download-submit',
-    _startDownload);
-
   JX.Stratcom.listen(
     'click',
     'embed-download-form',

From d5befb1a0ea3658feb88997cb9c016ce82492012 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Wed, 28 Feb 2018 18:51:37 -0800
Subject: [PATCH 29/39] Block use of "<base />" in the Content Security Policy

Summary: Ref T4340. We don't use "<base />" so we can safely block it.

Test Plan: Injected "<base />" into a page, saw an error in the console showing that the browser had blocked it.

Maniphest Tasks: T4340

Differential Revision: https://secure.phabricator.com/D19158
---
 src/aphront/response/AphrontResponse.php | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/aphront/response/AphrontResponse.php b/src/aphront/response/AphrontResponse.php
index ed3d6d3435..05d230aa71 100644
--- a/src/aphront/response/AphrontResponse.php
+++ b/src/aphront/response/AphrontResponse.php
@@ -154,6 +154,9 @@ abstract class AphrontResponse extends Phobject {
     // download URI instead of submitting a form to it.
     $csp[] = "form-action 'self'";
 
+    // Block use of "<base>" to change the origin of relative URIs on the page.
+    $csp[] = "base-uri 'none'";
+
     $csp = implode('; ', $csp);
 
     return $csp;

From 94d340fcffab6642e9ddc2e51dfa1a95f76f2c35 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Wed, 28 Feb 2018 19:07:19 -0800
Subject: [PATCH 30/39] Include OAuth targets in "form-action"
 Content-Security-Policy

Summary:
Ref T4340. Some "Register/Login" and "Link External Account" buttons are forms which submit to third-party sites. Whitelist these targets when pages render an OAuth form.

Safari, at least, also prevents a redirect to a third-party domain after a form submission to the local domain, so when we first redirect locally (as with Twitter and other OAuth1 providers) we need to authorize an additional URI.

Test Plan: Clicked all my registration buttons locally without hitting CSP issues.

Maniphest Tasks: T4340

Differential Revision: https://secure.phabricator.com/D19159
---
 src/aphront/response/AphrontResponse.php        | 17 +++++++++--------
 .../auth/provider/PhabricatorAuthProvider.php   | 11 +++++++++++
 .../provider/PhabricatorOAuth1AuthProvider.php  |  3 +++
 .../provider/PhortuneStripePaymentProvider.php  |  4 ++--
 .../control/AphrontFormRecaptchaControl.php     |  6 +++---
 src/view/page/PhabricatorStandardPageView.php   |  2 +-
 6 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/src/aphront/response/AphrontResponse.php b/src/aphront/response/AphrontResponse.php
index 05d230aa71..73c4e68f00 100644
--- a/src/aphront/response/AphrontResponse.php
+++ b/src/aphront/response/AphrontResponse.php
@@ -24,9 +24,10 @@ abstract class AphrontResponse extends Phobject {
   final public function addContentSecurityPolicyURI($kind, $uri) {
     if ($this->contentSecurityPolicyURIs === null) {
       $this->contentSecurityPolicyURIs = array(
-         'script' => array(),
-         'connect' => array(),
-         'frame' => array(),
+         'script-src' => array(),
+         'connect-src' => array(),
+         'frame-src' => array(),
+         'form-action' => array(),
        );
     }
 
@@ -125,14 +126,14 @@ abstract class AphrontResponse extends Phobject {
 
     // On a small number of pages, including the Stripe workflow and the
     // ReCAPTCHA challenge, we embed external Javascript directly.
-    $csp[] = $this->newContentSecurityPolicy('script', $default);
+    $csp[] = $this->newContentSecurityPolicy('script-src', $default);
 
     // We need to specify that we can connect to ourself in order for AJAX
     // requests to work.
-    $csp[] = $this->newContentSecurityPolicy('connect', "'self'");
+    $csp[] = $this->newContentSecurityPolicy('connect-src', "'self'");
 
     // DarkConsole and PHPAST both use frames to render some content.
-    $csp[] = $this->newContentSecurityPolicy('frame', "'self'");
+    $csp[] = $this->newContentSecurityPolicy('frame-src', "'self'");
 
     // This is a more modern flavor of of "X-Frame-Options" and prevents
     // clickjacking attacks where the page is included in a tiny iframe and
@@ -152,7 +153,7 @@ abstract class AphrontResponse extends Phobject {
     // This can result in some trickiness with file downloads if applications
     // try to start downloads by submitting a dialog. Redirect to the file's
     // download URI instead of submitting a form to it.
-    $csp[] = "form-action 'self'";
+    $csp[] = $this->newContentSecurityPolicy('form-action', "'self'");
 
     // Block use of "<base>" to change the origin of relative URIs on the page.
     $csp[] = "base-uri 'none'";
@@ -177,7 +178,7 @@ abstract class AphrontResponse extends Phobject {
     }
     $sources = array_unique($sources);
 
-    return "{$type}-src ".implode(' ', $sources);
+    return $type.' '.implode(' ', $sources);
   }
 
   private function newContentSecurityPolicySource($uri) {
diff --git a/src/applications/auth/provider/PhabricatorAuthProvider.php b/src/applications/auth/provider/PhabricatorAuthProvider.php
index d655efd790..0525edad54 100644
--- a/src/applications/auth/provider/PhabricatorAuthProvider.php
+++ b/src/applications/auth/provider/PhabricatorAuthProvider.php
@@ -447,6 +447,13 @@ abstract class PhabricatorAuthProvider extends Phobject {
         ));
     }
 
+    $static_response = CelerityAPI::getStaticResourceResponse();
+    $static_response->addContentSecurityPolicyURI('form-action', (string)$uri);
+
+    foreach ($this->getContentSecurityPolicyFormActions() as $csp_uri) {
+      $static_response->addContentSecurityPolicyURI('form-action', $csp_uri);
+    }
+
     return phabricator_form(
       $viewer,
       array(
@@ -505,4 +512,8 @@ abstract class PhabricatorAuthProvider extends Phobject {
     throw new PhutilMethodNotImplementedException();
   }
 
+  protected function getContentSecurityPolicyFormActions() {
+    return array();
+  }
+
 }
diff --git a/src/applications/auth/provider/PhabricatorOAuth1AuthProvider.php b/src/applications/auth/provider/PhabricatorOAuth1AuthProvider.php
index c62983de2f..2fbc637460 100644
--- a/src/applications/auth/provider/PhabricatorOAuth1AuthProvider.php
+++ b/src/applications/auth/provider/PhabricatorOAuth1AuthProvider.php
@@ -208,6 +208,9 @@ abstract class PhabricatorOAuth1AuthProvider
     parent::willRenderLinkedAccount($viewer, $item, $account);
   }
 
+  protected function getContentSecurityPolicyFormActions() {
+    return $this->getAdapter()->getContentSecurityPolicyFormActions();
+  }
 
 /* -(  Temporary Secrets  )-------------------------------------------------- */
 
diff --git a/src/applications/phortune/provider/PhortuneStripePaymentProvider.php b/src/applications/phortune/provider/PhortuneStripePaymentProvider.php
index 370f132a1f..bdaa4294b2 100644
--- a/src/applications/phortune/provider/PhortuneStripePaymentProvider.php
+++ b/src/applications/phortune/provider/PhortuneStripePaymentProvider.php
@@ -285,8 +285,8 @@ final class PhortuneStripePaymentProvider extends PhortunePaymentProvider {
       ->addScript($src);
 
     CelerityAPI::getStaticResourceResponse()
-      ->addContentSecurityPolicyURI('script', $src)
-      ->addContentSecurityPolicyURI('frame', $src);
+      ->addContentSecurityPolicyURI('script-src', $src)
+      ->addContentSecurityPolicyURI('frame-src', $src);
 
     Javelin::initBehavior(
       'stripe-payment-form',
diff --git a/src/view/form/control/AphrontFormRecaptchaControl.php b/src/view/form/control/AphrontFormRecaptchaControl.php
index b0559754e9..4152f7d8b7 100644
--- a/src/view/form/control/AphrontFormRecaptchaControl.php
+++ b/src/view/form/control/AphrontFormRecaptchaControl.php
@@ -43,9 +43,9 @@ final class AphrontFormRecaptchaControl extends AphrontFormControl {
     $pubkey = PhabricatorEnv::getEnvConfig('recaptcha.public-key');
 
     CelerityAPI::getStaticResourceResponse()
-      ->addContentSecurityPolicyURI('script', $js)
-      ->addContentSecurityPolicyURI('script', 'https://www.gstatic.com/')
-      ->addContentSecurityPolicyURI('frame', 'https://www.google.com/');
+      ->addContentSecurityPolicyURI('script-src', $js)
+      ->addContentSecurityPolicyURI('script-src', 'https://www.gstatic.com/')
+      ->addContentSecurityPolicyURI('frame-src', 'https://www.google.com/');
 
     return array(
       phutil_tag(
diff --git a/src/view/page/PhabricatorStandardPageView.php b/src/view/page/PhabricatorStandardPageView.php
index 8ee7566969..aca98b7205 100644
--- a/src/view/page/PhabricatorStandardPageView.php
+++ b/src/view/page/PhabricatorStandardPageView.php
@@ -584,7 +584,7 @@ final class PhabricatorStandardPageView extends PhabricatorBarePageView
           ) + $this->buildAphlictListenConfigData());
 
         CelerityAPI::getStaticResourceResponse()
-          ->addContentSecurityPolicyURI('connect', $client_uri);
+          ->addContentSecurityPolicyURI('connect-src', $client_uri);
       }
     }
 

From 4466402c5a6dd35e0f0a4fb58423b1f7dc10a618 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Thu, 1 Mar 2018 06:09:51 -0800
Subject: [PATCH 31/39] Move Paste line range reading code into AphrontRequest

Summary: Ref T13088. This lifts the code for parsing "$x-y" line ranges in URIs into AphrontRequest so Diffusion, Paste, Harbormaster, etc., can share it.

Test Plan: Viewed lines, line ranges, no lines, negative line ranges, line ranges with 0, and extremely long line ranges in Paste.

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19162
---
 src/aphront/AphrontRequest.php                | 48 +++++++++++++++++++
 .../PhabricatorPasteViewController.php        | 29 ++++-------
 2 files changed, 56 insertions(+), 21 deletions(-)

diff --git a/src/aphront/AphrontRequest.php b/src/aphront/AphrontRequest.php
index 9d806a851d..c1dc0d1305 100644
--- a/src/aphront/AphrontRequest.php
+++ b/src/aphront/AphrontRequest.php
@@ -49,6 +49,54 @@ final class AphrontRequest extends Phobject {
     return idx($this->uriData, $key, $default);
   }
 
+  /**
+   * Read line range parameter data from the request.
+   *
+   * Applications like Paste, Diffusion, and Harbormaster use "$12-14" in the
+   * URI to allow users to link to particular lines.
+   *
+   * @param string URI data key to pull line range information from.
+   * @param int|null Maximum length of the range.
+   * @return null|pair<int, int> Null, or beginning and end of the range.
+   */
+  public function getURILineRange($key, $limit) {
+    $range = $this->getURIData($key);
+    if (!strlen($range)) {
+      return null;
+    }
+
+    $range = explode('-', $range, 2);
+
+    foreach ($range as $key => $value) {
+      $value = (int)$value;
+      if (!$value) {
+        // If either value is "0", discard the range.
+        return null;
+      }
+      $range[$key] = $value;
+    }
+
+    // If the range is like "$10", treat it like "$10-10".
+    if (count($range) == 1) {
+      $range[] = head($range);
+    }
+
+    // If the range is "$7-5", treat it like "$5-7".
+    if ($range[1] < $range[0]) {
+      $range = array_reverse($range);
+    }
+
+    // If the user specified something like "$1-999999999" and we have a limit,
+    // clamp it to a more reasonable range.
+    if ($limit !== null) {
+      if ($range[1] - $range[0] > $limit) {
+        $range[1] = $range[0] + $limit;
+      }
+    }
+
+    return $range;
+  }
+
   public function setApplicationConfiguration(
     $application_configuration) {
     $this->applicationConfiguration = $application_configuration;
diff --git a/src/applications/paste/controller/PhabricatorPasteViewController.php b/src/applications/paste/controller/PhabricatorPasteViewController.php
index f29c93a162..9b4079fd46 100644
--- a/src/applications/paste/controller/PhabricatorPasteViewController.php
+++ b/src/applications/paste/controller/PhabricatorPasteViewController.php
@@ -2,30 +2,10 @@
 
 final class PhabricatorPasteViewController extends PhabricatorPasteController {
 
-  private $highlightMap;
-
   public function shouldAllowPublic() {
     return true;
   }
 
-  public function willProcessRequest(array $data) {
-    $raw_lines = idx($data, 'lines');
-    $map = array();
-    if ($raw_lines) {
-      $lines = explode('-', $raw_lines);
-      $first = idx($lines, 0, 0);
-      $last = idx($lines, 1);
-      if ($last) {
-        $min = min($first, $last);
-        $max = max($first, $last);
-        $map = array_fuse(range($min, $max));
-      } else {
-        $map[$first] = $first;
-      }
-    }
-    $this->highlightMap = $map;
-  }
-
   public function handleRequest(AphrontRequest $request) {
     $viewer = $request->getViewer();
     $id = $request->getURIData('id');
@@ -40,11 +20,18 @@ final class PhabricatorPasteViewController extends PhabricatorPasteController {
       return new Aphront404Response();
     }
 
+    $lines = $request->getURILineRange('lines', 1000);
+    if ($lines) {
+      $map = range($lines[0], $lines[1]);
+    } else {
+      $map = array();
+    }
+
     $header = $this->buildHeaderView($paste);
     $curtain = $this->buildCurtain($paste);
 
     $subheader = $this->buildSubheaderView($paste);
-    $source_code = $this->buildSourceCodeView($paste, $this->highlightMap);
+    $source_code = $this->buildSourceCodeView($paste, $map);
 
     require_celerity_resource('paste-css');
 

From 49af4165bc70b47b7fde68ff7b259fe187ea9d67 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Thu, 1 Mar 2018 06:43:24 -0800
Subject: [PATCH 32/39] Support rendering arbitrary sections in the middle of a
 Harbormaster build log so links to line 3500 work

Summary:
Depends on D19162. Ref T13088. When a user links to `$1234`, we need to render a default view of the log with a piece at the head, a piece at the end, and a piece in the middle.

We also need to figure out the offset for line 1234, or multiple offsets for "1234-2345".

Since the logic views/reads mostly anticipated this it isn't too much of a mess, although there are a couple of bugs this exposes with view specifications that use combinations of parameters which were previously impossible.

Test Plan: Viewed a large log with no line marker. Viewed `$1`. Viewed `$end`. Viewed `$35-40`, etc. Expanded context around logs.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19163
---
 resources/celerity/map.php                    |  10 +-
 .../HarbormasterBuildLogRenderController.php  | 190 +++++++++++++++---
 .../storage/build/HarbormasterBuildLog.php    |  30 +++
 .../view/HarbormasterBuildLogView.php         |   3 +-
 .../harbormaster/behavior-harbormaster-log.js |   2 +-
 5 files changed, 202 insertions(+), 33 deletions(-)

diff --git a/resources/celerity/map.php b/resources/celerity/map.php
index 4eebc07609..9b68b42952 100644
--- a/resources/celerity/map.php
+++ b/resources/celerity/map.php
@@ -416,7 +416,7 @@ return array(
     'rsrc/js/application/drydock/drydock-live-operation-status.js' => '901935ef',
     'rsrc/js/application/files/behavior-icon-composer.js' => '8499b6ab',
     'rsrc/js/application/files/behavior-launch-icon-composer.js' => '48086888',
-    'rsrc/js/application/harbormaster/behavior-harbormaster-log.js' => '796a8803',
+    'rsrc/js/application/harbormaster/behavior-harbormaster-log.js' => 'ab1173d1',
     'rsrc/js/application/herald/HeraldRuleEditor.js' => 'dca75c0e',
     'rsrc/js/application/herald/PathTypeahead.js' => 'f7fc67ec',
     'rsrc/js/application/herald/herald-rule-editor.js' => '7ebaeed3',
@@ -636,7 +636,7 @@ return array(
     'javelin-behavior-event-all-day' => 'b41537c9',
     'javelin-behavior-fancy-datepicker' => 'ecf4e799',
     'javelin-behavior-global-drag-and-drop' => '960f6a39',
-    'javelin-behavior-harbormaster-log' => '796a8803',
+    'javelin-behavior-harbormaster-log' => 'ab1173d1',
     'javelin-behavior-herald-rule-editor' => '7ebaeed3',
     'javelin-behavior-high-security-warning' => 'a464fe03',
     'javelin-behavior-history-install' => '7ee2b591',
@@ -1526,9 +1526,6 @@ return array(
       'javelin-behavior',
       'javelin-quicksand',
     ),
-    '796a8803' => array(
-      'javelin-behavior',
-    ),
     '7a68dda3' => array(
       'owners-path-editor',
       'javelin-behavior',
@@ -1768,6 +1765,9 @@ return array(
       'javelin-util',
       'phabricator-prefab',
     ),
+    'ab1173d1' => array(
+      'javelin-behavior',
+    ),
     'ab2f381b' => array(
       'javelin-request',
       'javelin-behavior',
diff --git a/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
index 6d623f2bc5..27a75f288b 100644
--- a/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
+++ b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
@@ -16,6 +16,8 @@ final class HarbormasterBuildLogRenderController
       return new Aphront404Response();
     }
 
+    $highlight_range = $request->getURILineRange('lines', 1000);
+
     $log_size = $this->getTotalByteLength($log);
 
     $head_lines = $request->getInt('head');
@@ -65,6 +67,16 @@ final class HarbormasterBuildLogRenderController
       );
     }
 
+    if ($highlight_range) {
+      $highlight_views = $this->getHighlightViews(
+        $log,
+        $highlight_range,
+        $log_size);
+      foreach ($highlight_views as $highlight_view) {
+        $views[] = $highlight_view;
+      }
+    }
+
     if ($tail_lines > 0) {
       $views[] = array(
         'offset' => $tail_offset,
@@ -86,10 +98,11 @@ final class HarbormasterBuildLogRenderController
 
       $direction = $read['direction'];
       if ($direction < 0) {
-        $offset -= $read_length;
-        if ($offset < 0) {
+        if ($offset > $read_length) {
+          $offset -= $read_length;
+        } else {
+          $read_length = $offset;
           $offset = 0;
-          $read_length = $log_size;
         }
       }
 
@@ -215,8 +228,10 @@ final class HarbormasterBuildLogRenderController
         }
 
         $limit = $view['limit'];
-        if ($limit < ($view_offset + $view_length)) {
-          $view_length = ($limit - $view_offset);
+        if ($limit !== null) {
+          if ($limit < ($view_offset + $view_length)) {
+            $view_length = ($limit - $view_offset);
+          }
         }
       } else {
         $view_offset = $data_offset;
@@ -226,9 +241,11 @@ final class HarbormasterBuildLogRenderController
         }
 
         $limit = $view['limit'];
-        if ($limit > $view_offset) {
-          $view_length -= ($limit - $view_offset);
-          $view_offset = $limit;
+        if ($limit !== null) {
+          if ($limit > $view_offset) {
+            $view_length -= ($limit - $view_offset);
+            $view_offset = $limit;
+          }
         }
       }
 
@@ -325,7 +342,6 @@ final class HarbormasterBuildLogRenderController
     }
 
     $uri = $log->getURI();
-    $highlight_range = $request->getURIData('lines');
 
     $rows = array();
     foreach ($render as $range) {
@@ -339,6 +355,16 @@ final class HarbormasterBuildLogRenderController
         $display_line = ($line['line'] + 1);
         $display_text = ($line['data']);
 
+        $cell_attr = array();
+        if ($highlight_range) {
+          if (($display_line >= $highlight_range[0]) &&
+              ($display_line <= $highlight_range[1])) {
+            $cell_attr = array(
+              'class' => 'phabricator-source-highlight',
+            );
+          }
+        }
+
         $display_line = phutil_tag(
           'a',
           array(
@@ -347,7 +373,7 @@ final class HarbormasterBuildLogRenderController
           $display_line);
 
         $line_cell = phutil_tag('th', array(), $display_line);
-        $text_cell = phutil_tag('td', array(), $display_text);
+        $text_cell = phutil_tag('td', $cell_attr, $display_text);
 
         $rows[] = phutil_tag(
           'tr',
@@ -557,25 +583,43 @@ final class HarbormasterBuildLogRenderController
         $vs = $vview['viewOffset'];
         $ve = $vs + $vview['viewLength'];
 
+        // Don't merge if one of the slices starts at a byte offset
+        // significantly after the other ends.
+        if (($vs > $ue + $body_bytes) || ($us > $ve + $body_bytes)) {
+          continue;
+        }
+
         $uss = $uview['sliceOffset'];
         $use = $uss + $uview['sliceLength'];
 
         $vss = $vview['sliceOffset'];
         $vse = $vss + $vview['sliceLength'];
 
-        if ($ue <= $vs) {
-          if (($ue + $body_bytes) >= $vs) {
-            if (($use + $body_lines) >= $vss) {
-              $views[$ukey] = array(
-                'sliceLength' => ($vse - $uss),
-                'viewLength' => ($ve - $us),
-              ) + $views[$ukey];
-
-              unset($views[$vkey]);
-              continue;
-            }
-          }
+        // Don't merge if one of the slices starts at a line offset
+        // significantly after the other ends.
+        if ($uss > ($vse + $body_lines) || $vss > ($use + $body_lines)) {
+          continue;
         }
+
+        // These views are overlapping or nearly overlapping, so we merge
+        // them. We merge views even if they aren't exactly adjacent since
+        // it's silly to render an "expand more" which only expands a couple
+        // of lines.
+
+        $offset = min($us, $vs);
+        $length = max($ue, $ve) - $offset;
+
+        $slice_offset = min($uss, $vss);
+        $slice_length = max($use, $vse) - $slice_offset;
+
+        $views[$ukey] = array(
+          'viewOffset' => $offset,
+          'viewLength' => $length,
+          'sliceOffset' => $slice_offset,
+          'sliceLength' => $slice_length,
+        ) + $views[$ukey];
+
+        unset($views[$vkey]);
       }
     }
 
@@ -603,7 +647,7 @@ final class HarbormasterBuildLogRenderController
         'meta' => array(
           'headOffset' => $range['head'],
           'tailOffset' => $range['tail'],
-          'head' => 4,
+          'head' => 128,
           'tail' => 0,
         ),
       ),
@@ -620,8 +664,8 @@ final class HarbormasterBuildLogRenderController
         'meta' => array(
           'headOffset' => $range['head'],
           'tailOffset' => $range['tail'],
-          'head' => 2,
-          'tail' => 2,
+          'head' => 128,
+          'tail' => 128,
         ),
       ),
       $mid_text);
@@ -640,7 +684,7 @@ final class HarbormasterBuildLogRenderController
           'headOffset' => $range['head'],
           'tailOffset' => $range['tail'],
           'head' => 0,
-          'tail' => 4,
+          'tail' => 128,
         ),
       ),
       $down_text);
@@ -727,4 +771,98 @@ final class HarbormasterBuildLogRenderController
     return phutil_tag('tr', array(), $format_cells);
   }
 
+  private function getHighlightViews(
+    HarbormasterBuildLog $log,
+    array $range,
+    $log_size) {
+    // If we're highlighting a line range in the file, we first need to figure
+    // out the offsets for the lines we care about.
+    list($range_min, $range_max) = $range;
+
+    // Read the markers to find a range we can load which includes both lines.
+    $read_range = $log->getLineSpanningRange($range_min, $range_max);
+    list($min_pos, $max_pos, $min_line) = $read_range;
+
+    $length = ($max_pos - $min_pos);
+
+    // Reject to do the read if it requires us to examine a huge amount of
+    // data. For example, the user may request lines "$1-1000" of a file where
+    // each line has 100MB of text.
+    $limit = (1024 * 1024 * 16);
+    if ($length > $limit) {
+      return array();
+    }
+
+    $data = $log->loadData($min_pos, $length);
+
+    $offset = $min_pos;
+    $min_offset = null;
+    $max_offset = null;
+
+    $lines = $this->getLines($data);
+    $number = ($min_line + 1);
+
+    foreach ($lines as $line) {
+      if ($min_offset === null) {
+        if ($number === $range_min) {
+          $min_offset = $offset;
+        }
+      }
+
+      $offset += strlen($line);
+
+      if ($max_offset === null) {
+        if ($number === $range_max) {
+          $max_offset = $offset;
+          break;
+        }
+      }
+
+      $number += 1;
+    }
+
+    $context_lines = 8;
+
+    // Build views around the beginning and ends of the respective lines. We
+    // expect these views to overlap significantly in normal circumstances
+    // and be merged later.
+    $views = array();
+
+    if ($min_offset !== null) {
+      $views[] = array(
+        'offset' => $min_offset,
+        'lines' => $context_lines + ($range_max - $range_min) - 1,
+        'direction' => 1,
+        'limit' => null,
+      );
+      if ($min_offset > 0) {
+        $views[] = array(
+          'offset' => $min_offset,
+          'lines' => $context_lines,
+          'direction' => -1,
+          'limit' => null,
+        );
+      }
+    }
+
+    if ($max_offset !== null) {
+      $views[] = array(
+        'offset' => $max_offset,
+        'lines' => $context_lines + ($range_max - $range_min),
+        'direction' => -1,
+        'limit' => null,
+      );
+      if ($max_offset < $log_size) {
+        $views[] = array(
+          'offset' => $max_offset,
+          'lines' => $context_lines,
+          'direction' => 1,
+          'limit' => null,
+        );
+      }
+    }
+
+    return $views;
+  }
+
 }
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
index bf69457254..5bc358ccc8 100644
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
@@ -212,6 +212,36 @@ final class HarbormasterBuildLog
     return $parts;
   }
 
+  public function getLineSpanningRange($min_line, $max_line) {
+    $map = $this->getLineMap();
+    if (!$map) {
+      throw new Exception(pht('No line map.'));
+    }
+
+    $min_pos = 0;
+    $min_line = 0;
+    $max_pos = $this->getByteLength();
+    list($map) = $map;
+    foreach ($map as $marker) {
+      list($offset, $count) = $marker;
+
+      if ($count < $min_line) {
+        if ($offset > $min_pos) {
+          $min_pos = $offset;
+          $min_line = $count;
+        }
+      }
+
+      if ($count > $max_line) {
+        $max_pos = min($max_pos, $offset);
+        break;
+      }
+    }
+
+    return array($min_pos, $max_pos, $min_line);
+  }
+
+
   public function getReadPosition($read_offset) {
     $position = array(0, 0);
 
diff --git a/src/applications/harbormaster/view/HarbormasterBuildLogView.php b/src/applications/harbormaster/view/HarbormasterBuildLogView.php
index 7200cdce70..768c55a504 100644
--- a/src/applications/harbormaster/view/HarbormasterBuildLogView.php
+++ b/src/applications/harbormaster/view/HarbormasterBuildLogView.php
@@ -67,7 +67,8 @@ final class HarbormasterBuildLogView extends AphrontView {
         'harbormaster-log',
         array(
           'contentNodeID' => $content_id,
-          'renderURI' => $log->getRenderURI($this->getHighlightedLineRange()),
+          'initialURI' => $log->getRenderURI($this->getHighlightedLineRange()),
+          'renderURI' => $log->getRenderURI(null),
         ));
 
       $box_view->appendChild($content_div);
diff --git a/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js b/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js
index 510631e1e0..2b91106725 100644
--- a/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js
+++ b/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js
@@ -79,7 +79,7 @@ JX.behavior('harbormaster-log', function(config) {
     JX.DOM.setContent(contentNode, JX.$H(r.markup));
   }
 
-  var uri = new JX.URI(config.renderURI);
+  var uri = new JX.URI(config.initialURI);
 
   new JX.Request(uri, onresponse)
     .send();

From fe3de5dd58b4b7ae3b696dd250c837971682cd96 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Thu, 1 Mar 2018 07:43:53 -0800
Subject: [PATCH 33/39] Make Paste source code line highlighting behavior more
 generic

Summary: Depends on D19163. Ref T13088. Increase the generality of this code so it can be shared with Harbormaster.

Test Plan: Clicked individual lines, clicked-and-dragged, etc., in Paste. Got sensible URI and highlight behaviors.

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19164
---
 resources/celerity/map.php                    |  20 +--
 src/view/layout/PhabricatorSourceCodeView.php |  14 +-
 .../layout/phabricator-source-code-view.css   |  11 +-
 webroot/rsrc/js/core/behavior-line-linker.js  | 122 +++++++++++++-----
 4 files changed, 117 insertions(+), 50 deletions(-)

diff --git a/resources/celerity/map.php b/resources/celerity/map.php
index 9b68b42952..f636013c3b 100644
--- a/resources/celerity/map.php
+++ b/resources/celerity/map.php
@@ -122,7 +122,7 @@ return array(
     'rsrc/css/font/font-lato.css' => 'c7ccd872',
     'rsrc/css/font/phui-font-icon-base.css' => '870a7360',
     'rsrc/css/layout/phabricator-filetree-view.css' => 'b912ad97',
-    'rsrc/css/layout/phabricator-source-code-view.css' => 'aea41829',
+    'rsrc/css/layout/phabricator-source-code-view.css' => '926ced2d',
     'rsrc/css/phui/button/phui-button-bar.css' => 'f1ff5494',
     'rsrc/css/phui/button/phui-button-simple.css' => '8e1baf68',
     'rsrc/css/phui/button/phui-button.css' => '1863cc6e',
@@ -495,7 +495,7 @@ return array(
     'rsrc/js/core/behavior-keyboard-pager.js' => 'a8da01f0',
     'rsrc/js/core/behavior-keyboard-shortcuts.js' => '01fca1f0',
     'rsrc/js/core/behavior-lightbox-attachments.js' => 'e31fad01',
-    'rsrc/js/core/behavior-line-linker.js' => '1499a8cb',
+    'rsrc/js/core/behavior-line-linker.js' => 'c479ac01',
     'rsrc/js/core/behavior-more.js' => 'a80d0378',
     'rsrc/js/core/behavior-object-selector.js' => '77c1f0b0',
     'rsrc/js/core/behavior-oncopy.js' => '2926fff2',
@@ -658,7 +658,7 @@ return array(
     'javelin-behavior-phabricator-gesture-example' => '558829c2',
     'javelin-behavior-phabricator-keyboard-pager' => 'a8da01f0',
     'javelin-behavior-phabricator-keyboard-shortcuts' => '01fca1f0',
-    'javelin-behavior-phabricator-line-linker' => '1499a8cb',
+    'javelin-behavior-phabricator-line-linker' => 'c479ac01',
     'javelin-behavior-phabricator-nav' => '836f966d',
     'javelin-behavior-phabricator-notification-example' => '8ce821c5',
     'javelin-behavior-phabricator-object-selector' => '77c1f0b0',
@@ -802,7 +802,7 @@ return array(
     'phabricator-search-results-css' => '505dd8cf',
     'phabricator-shaped-request' => '7cbe244b',
     'phabricator-slowvote-css' => 'a94b7230',
-    'phabricator-source-code-view-css' => 'aea41829',
+    'phabricator-source-code-view-css' => '926ced2d',
     'phabricator-standard-page-view' => '34ee718b',
     'phabricator-textareautils' => '320810c8',
     'phabricator-title' => '485aaa6c',
@@ -998,12 +998,6 @@ return array(
       'javelin-dom',
       'javelin-typeahead-normalizer',
     ),
-    '1499a8cb' => array(
-      'javelin-behavior',
-      'javelin-stratcom',
-      'javelin-dom',
-      'javelin-history',
-    ),
     '15d5ff71' => array(
       'aphront-typeahead-control-css',
       'phui-tag-view-css',
@@ -1937,6 +1931,12 @@ return array(
       'javelin-stratcom',
       'phabricator-tooltip',
     ),
+    'c479ac01' => array(
+      'javelin-behavior',
+      'javelin-stratcom',
+      'javelin-dom',
+      'javelin-history',
+    ),
     'c587b80f' => array(
       'javelin-install',
     ),
diff --git a/src/view/layout/PhabricatorSourceCodeView.php b/src/view/layout/PhabricatorSourceCodeView.php
index eb34925b3f..f5f6c22b51 100644
--- a/src/view/layout/PhabricatorSourceCodeView.php
+++ b/src/view/layout/PhabricatorSourceCodeView.php
@@ -73,6 +73,7 @@ final class PhabricatorSourceCodeView extends AphrontView {
           pht('...')));
     }
 
+    $base_uri = (string)$this->uri;
     foreach ($lines as $line) {
 
       // NOTE: See phabricator-oncopy behavior.
@@ -84,17 +85,16 @@ final class PhabricatorSourceCodeView extends AphrontView {
       }
 
       if ($this->canClickHighlight) {
-        $line_uri = $this->uri.'$'.$line_number;
-        $line_href = (string)new PhutilURI($line_uri);
+        $line_href = $base_uri.'$'.$line_number;
 
-        $tag_number = javelin_tag(
+        $tag_number = phutil_tag(
           'a',
           array(
             'href' => $line_href,
           ),
           $line_number);
       } else {
-        $tag_number = javelin_tag(
+        $tag_number = phutil_tag(
           'span',
           array(),
           $line_number);
@@ -104,11 +104,10 @@ final class PhabricatorSourceCodeView extends AphrontView {
         'tr',
         $row_attributes,
         array(
-          javelin_tag(
+          phutil_tag(
             'th',
             array(
               'class' => 'phabricator-source-line',
-              'sigil' => 'phabricator-source-line',
             ),
             $tag_number),
           phutil_tag(
@@ -134,6 +133,9 @@ final class PhabricatorSourceCodeView extends AphrontView {
         array(
           'class' => implode(' ', $classes),
           'sigil' => 'phabricator-source',
+          'meta' => array(
+            'uri' => (string)$this->uri,
+          ),
         ),
         phutil_implode_html('', $rows)));
   }
diff --git a/webroot/rsrc/css/layout/phabricator-source-code-view.css b/webroot/rsrc/css/layout/phabricator-source-code-view.css
index a855e01447..446366cdf1 100644
--- a/webroot/rsrc/css/layout/phabricator-source-code-view.css
+++ b/webroot/rsrc/css/layout/phabricator-source-code-view.css
@@ -31,7 +31,6 @@
 .phabricator-source-line {
   background-color: {$paste.highlight};
   text-align: right;
-  padding: 2px 6px 1px 12px;
   border-right: 1px solid {$paste.border};
   color: {$sh-yellowtext};
 
@@ -48,17 +47,23 @@
 
 th.phabricator-source-line a {
   color: {$darkbluetext};
+  display: block;
+  padding: 2px 6px 1px 12px;
 }
 
-th.phabricator-source-line:hover {
+th.phabricator-source-line a:hover {
   background: {$paste.border};
-  cursor: pointer;
+  text-decoration: none;
 }
 
 .phabricator-source-highlight {
   background: {$paste.highlight};
 }
 
+.phabricator-source-highlight th.phabricator-source-line {
+  background: {$paste.border};
+}
+
 .phabricator-source-code-summary {
   padding-bottom: 8px;
 }
diff --git a/webroot/rsrc/js/core/behavior-line-linker.js b/webroot/rsrc/js/core/behavior-line-linker.js
index 0a651d6ca0..ed93e6e11b 100644
--- a/webroot/rsrc/js/core/behavior-line-linker.js
+++ b/webroot/rsrc/js/core/behavior-line-linker.js
@@ -10,6 +10,7 @@ JX.behavior('phabricator-line-linker', function() {
   var origin = null;
   var target = null;
   var root = null;
+  var highlighted = null;
 
   var editor_link = null;
   try {
@@ -19,48 +20,101 @@ JX.behavior('phabricator-line-linker', function() {
   }
 
   function getRowNumber(tr) {
-    var th = JX.DOM.find(tr, 'th', 'phabricator-source-line');
+    var th = tr.firstChild;
     return +(th.textContent || th.innerText);
   }
 
   JX.Stratcom.listen(
-    'mousedown',
-    'phabricator-source-line',
+    ['click', 'mousedown'],
+    ['phabricator-source', 'tag:tr', 'tag:th', 'tag:a'],
     function(e) {
       if (!e.isNormalMouseEvent()) {
         return;
       }
-      origin = e.getNode('tag:tr');
-      target = origin;
-      root = e.getNode('phabricator-source');
-      e.kill();
-    });
 
-  JX.Stratcom.listen(
-    'click',
-    'phabricator-source-line',
-    function(e) {
+      // Make sure the link we clicked is actually a line number in a source
+      // table, not some kind of link in some element embedded inside the
+      // table. The row's immediate ancestor table needs to be the table with
+      // the "phabricator-source" sigil.
+
+      var row = e.getNode('tag:tr');
+      var table = e.getNode('phabricator-source');
+      if (JX.DOM.findAbove(row, 'table') !== table) {
+        return;
+      }
+
+      var number = getRowNumber(row);
+      if (!number) {
+        return;
+      }
+
       e.kill();
+
+      // If this is a click event, kill it. We handle mousedown and mouseup
+      // instead.
+      if (e.getType() === 'click') {
+        return;
+      }
+
+      origin = row;
+      target = origin;
+
+      root = table;
     });
 
   var highlight = function(e) {
-    if (!origin || e.getNode('phabricator-source') !== root) {
+    if (!origin) {
+      return;
+    }
+
+    if (e.getNode('phabricator-source') !== root) {
       return;
     }
     target = e.getNode('tag:tr');
 
-    var highlighting = false;
-    var source = null;
-    var trs = JX.DOM.scry(root, 'tr');
-    for (var i = 0; i < trs.length; i++) {
-      if (!highlighting && (trs[i] === origin || trs[i] === target)) {
-        highlighting = true;
-        source = trs[i];
+    var min;
+    var max;
+
+    // NOTE: We're using position to figure out which order these rows are in,
+    // not row numbers. We do this because Harbormaster build logs may have
+    // multiple rows with the same row number.
+
+    if (JX.$V(origin).y <= JX.$V(target).y) {
+      min = origin;
+      max = target;
+    } else {
+      min = target;
+      max = origin;
+    }
+
+    // If we haven't changed highlighting, we don't have a list of highlighted
+    // nodes yet. Assume every row is highlighted.
+    var ii;
+    if (highlighted === null) {
+      highlighted = [];
+      var rows = JX.DOM.scry(root, 'tr');
+      for (ii = 0; ii < rows.length; ii++) {
+        highlighted.push(rows[ii]);
       }
-      JX.DOM.alterClass(trs[i], 'phabricator-source-highlight', highlighting);
-      if (trs[i] === (source === origin ? target : origin)) {
-        highlighting = false;
+    }
+
+    // Unhighlight any existing highlighted rows.
+    for (ii = 0; ii < highlighted.length; ii++) {
+      JX.DOM.alterClass(highlighted[ii], 'phabricator-source-highlight', false);
+    }
+    highlighted = [];
+
+    // Highlight the newly selected rows.
+    var cursor = min;
+    while (true) {
+      JX.DOM.alterClass(cursor, 'phabricator-source-highlight', true);
+      highlighted.push(cursor);
+
+      if (cursor === max) {
+        break;
       }
+
+      cursor = cursor.nextSibling;
     }
   };
 
@@ -75,20 +129,26 @@ JX.behavior('phabricator-line-linker', function() {
       }
 
       highlight(e);
+      e.kill();
 
       var o = getRowNumber(origin);
       var t = getRowNumber(target);
-      var lines = (o == t ? o : Math.min(o, t) + '-' + Math.max(o, t));
-      var th = JX.DOM.find(origin, 'th', 'phabricator-source-line');
-      var uri = JX.DOM.find(th, 'a').href;
-      uri = uri.replace(/(.*\$)\d+/, '$1' + lines);
+      var uri = JX.Stratcom.getData(root).uri;
+
       origin = null;
       target = null;
-      e.kill();
+      root = null;
+
+      var lines = (o == t ? o : Math.min(o, t) + '-' + Math.max(o, t));
+      uri = uri + '$' + lines;
+
       JX.History.replace(uri);
-      if (editor_link.href) {
-        var editdata = JX.Stratcom.getData(editor_link);
-        editor_link.href = editdata.link_template.replace('%25l', o);
+
+      if (editor_link) {
+        if (editor_link.href) {
+          var editdata = JX.Stratcom.getData(editor_link);
+          editor_link.href = editdata.link_template.replace('%25l', o);
+        }
       }
     });
 

From 73619c4643c9e3c9111294eb93d779cef4c5bdeb Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Thu, 1 Mar 2018 08:02:32 -0800
Subject: [PATCH 34/39] Share the Paste line highlighting behavior for
 Harbormaster build logs

Summary: Depends on D19164. Ref T13088. Now that the JS behaviors are generic, use them on the Harbormaster standalone page.

Test Plan: Clicked lines and dragged across line ranges. Reloaded pages. Saw expected highlighting behavior in the client and on the server across reloads.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19165
---
 resources/celerity/map.php                         |  4 ++--
 .../HarbormasterBuildLogRenderController.php       | 14 +++++++++-----
 .../HarbormasterBuildLogViewController.php         |  3 ++-
 .../harbormaster/view/HarbormasterBuildLogView.php | 10 ++++++++++
 .../css/application/harbormaster/harbormaster.css  |  6 +++++-
 5 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/resources/celerity/map.php b/resources/celerity/map.php
index f636013c3b..112c75919b 100644
--- a/resources/celerity/map.php
+++ b/resources/celerity/map.php
@@ -78,7 +78,7 @@ return array(
     'rsrc/css/application/feed/feed.css' => 'ecd4ec57',
     'rsrc/css/application/files/global-drag-and-drop.css' => 'b556a948',
     'rsrc/css/application/flag/flag.css' => 'bba8f811',
-    'rsrc/css/application/harbormaster/harbormaster.css' => '5dd4c2de',
+    'rsrc/css/application/harbormaster/harbormaster.css' => '2999ccaa',
     'rsrc/css/application/herald/herald-test.css' => 'a52e323e',
     'rsrc/css/application/herald/herald.css' => 'cd8d0134',
     'rsrc/css/application/maniphest/report.css' => '9b9580b7',
@@ -579,7 +579,7 @@ return array(
     'font-fontawesome' => 'e838e088',
     'font-lato' => 'c7ccd872',
     'global-drag-and-drop-css' => 'b556a948',
-    'harbormaster-css' => '5dd4c2de',
+    'harbormaster-css' => '2999ccaa',
     'herald-css' => 'cd8d0134',
     'herald-rule-editor' => 'dca75c0e',
     'herald-test-css' => 'a52e323e',
diff --git a/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
index 27a75f288b..4fa826b690 100644
--- a/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
+++ b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
@@ -355,11 +355,11 @@ final class HarbormasterBuildLogRenderController
         $display_line = ($line['line'] + 1);
         $display_text = ($line['data']);
 
-        $cell_attr = array();
+        $row_attr = array();
         if ($highlight_range) {
           if (($display_line >= $highlight_range[0]) &&
               ($display_line <= $highlight_range[1])) {
-            $cell_attr = array(
+            $row_attr = array(
               'class' => 'phabricator-source-highlight',
             );
           }
@@ -373,11 +373,11 @@ final class HarbormasterBuildLogRenderController
           $display_line);
 
         $line_cell = phutil_tag('th', array(), $display_line);
-        $text_cell = phutil_tag('td', $cell_attr, $display_text);
+        $text_cell = phutil_tag('td', array(), $display_text);
 
         $rows[] = phutil_tag(
           'tr',
-          array(),
+          $row_attr,
           array(
             $line_cell,
             $text_cell,
@@ -401,10 +401,14 @@ final class HarbormasterBuildLogRenderController
       }
     }
 
-    $table = phutil_tag(
+    $table = javelin_tag(
       'table',
       array(
         'class' => 'harbormaster-log-table PhabricatorMonospaced',
+        'sigil' => 'phabricator-source',
+        'meta' => array(
+          'uri' => $log->getURI(),
+        ),
       ),
       $rows);
 
diff --git a/src/applications/harbormaster/controller/HarbormasterBuildLogViewController.php b/src/applications/harbormaster/controller/HarbormasterBuildLogViewController.php
index 40641795c5..888b3bddb2 100644
--- a/src/applications/harbormaster/controller/HarbormasterBuildLogViewController.php
+++ b/src/applications/harbormaster/controller/HarbormasterBuildLogViewController.php
@@ -24,7 +24,8 @@ final class HarbormasterBuildLogViewController
     $log_view = id(new HarbormasterBuildLogView())
       ->setViewer($viewer)
       ->setBuildLog($log)
-      ->setHighlightedLineRange($request->getURIData('lines'));
+      ->setHighlightedLineRange($request->getURIData('lines'))
+      ->setEnableHighlighter(true);
 
     $crumbs = $this->buildApplicationCrumbs()
       ->addTextCrumb(pht('Build Logs'))
diff --git a/src/applications/harbormaster/view/HarbormasterBuildLogView.php b/src/applications/harbormaster/view/HarbormasterBuildLogView.php
index 768c55a504..5eb32a84e5 100644
--- a/src/applications/harbormaster/view/HarbormasterBuildLogView.php
+++ b/src/applications/harbormaster/view/HarbormasterBuildLogView.php
@@ -4,6 +4,7 @@ final class HarbormasterBuildLogView extends AphrontView {
 
   private $log;
   private $highlightedLineRange;
+  private $enableHighlighter;
 
   public function setBuildLog(HarbormasterBuildLog $log) {
     $this->log = $log;
@@ -23,6 +24,11 @@ final class HarbormasterBuildLogView extends AphrontView {
     return $this->highlightedLineRange;
   }
 
+  public function setEnableHighlighter($enable) {
+    $this->enableHighlighter = $enable;
+    return $this;
+  }
+
   public function render() {
     $viewer = $this->getViewer();
     $log = $this->getBuildLog();
@@ -50,6 +56,10 @@ final class HarbormasterBuildLogView extends AphrontView {
       ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
       ->setHeader($header);
 
+    if ($this->enableHighlighter) {
+      Javelin::initBehavior('phabricator-line-linker');
+    }
+
     $has_linemap = $log->getLineMap();
     if ($has_linemap) {
       $content_id = celerity_generate_unique_node_id();
diff --git a/webroot/rsrc/css/application/harbormaster/harbormaster.css b/webroot/rsrc/css/application/harbormaster/harbormaster.css
index b22658219a..0e8574464d 100644
--- a/webroot/rsrc/css/application/harbormaster/harbormaster.css
+++ b/webroot/rsrc/css/application/harbormaster/harbormaster.css
@@ -69,7 +69,11 @@
   padding: 4px 0;
 }
 
-.harbormaster-log-table tr.harbormaster-log-highlighted > td {
+.harbormaster-log-table tr.phabricator-source-highlight > th {
+  background: {$paste.border};
+}
+
+.harbormaster-log-table tr.phabricator-source-highlight > td {
   background: {$paste.highlight};
 }
 

From 4e91ad276d8ab45dc7ec26ebf72cc43ba945c892 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Thu, 1 Mar 2018 08:14:17 -0800
Subject: [PATCH 35/39] Prevent copying Harbormaster build log line numbers
 with CSS psuedocontent instead of ZWS

Summary:
Depends on D19165. Ref T13088. Currently, in other applications, we use Zero Width Spaces and Javascript "copy" listeners to prevent line numbers from being copied. This isn't terribly elegant.

Modern browsers support a second approach: using psuedo-elements with `content`. Try this in Harbormaster since it's conceptually cleaner, at least. One immediate drawback is that Command-F can't find this text either.

Test Plan: In Safari, Chrome and Firefox, highlighted ranges of lines and copy/pasted text. Got just text (no line numbers) in all cases.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19166
---
 resources/celerity/map.php                    | 20 +++++++++----------
 .../HarbormasterBuildLogRenderController.php  |  3 ++-
 .../application/harbormaster/harbormaster.css |  6 ++++++
 webroot/rsrc/js/core/behavior-line-linker.js  | 10 ++++++++++
 4 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/resources/celerity/map.php b/resources/celerity/map.php
index 112c75919b..3ffa4bb058 100644
--- a/resources/celerity/map.php
+++ b/resources/celerity/map.php
@@ -78,7 +78,7 @@ return array(
     'rsrc/css/application/feed/feed.css' => 'ecd4ec57',
     'rsrc/css/application/files/global-drag-and-drop.css' => 'b556a948',
     'rsrc/css/application/flag/flag.css' => 'bba8f811',
-    'rsrc/css/application/harbormaster/harbormaster.css' => '2999ccaa',
+    'rsrc/css/application/harbormaster/harbormaster.css' => 'cd73d427',
     'rsrc/css/application/herald/herald-test.css' => 'a52e323e',
     'rsrc/css/application/herald/herald.css' => 'cd8d0134',
     'rsrc/css/application/maniphest/report.css' => '9b9580b7',
@@ -495,7 +495,7 @@ return array(
     'rsrc/js/core/behavior-keyboard-pager.js' => 'a8da01f0',
     'rsrc/js/core/behavior-keyboard-shortcuts.js' => '01fca1f0',
     'rsrc/js/core/behavior-lightbox-attachments.js' => 'e31fad01',
-    'rsrc/js/core/behavior-line-linker.js' => 'c479ac01',
+    'rsrc/js/core/behavior-line-linker.js' => 'a9b946f8',
     'rsrc/js/core/behavior-more.js' => 'a80d0378',
     'rsrc/js/core/behavior-object-selector.js' => '77c1f0b0',
     'rsrc/js/core/behavior-oncopy.js' => '2926fff2',
@@ -579,7 +579,7 @@ return array(
     'font-fontawesome' => 'e838e088',
     'font-lato' => 'c7ccd872',
     'global-drag-and-drop-css' => 'b556a948',
-    'harbormaster-css' => '2999ccaa',
+    'harbormaster-css' => 'cd73d427',
     'herald-css' => 'cd8d0134',
     'herald-rule-editor' => 'dca75c0e',
     'herald-test-css' => 'a52e323e',
@@ -658,7 +658,7 @@ return array(
     'javelin-behavior-phabricator-gesture-example' => '558829c2',
     'javelin-behavior-phabricator-keyboard-pager' => 'a8da01f0',
     'javelin-behavior-phabricator-keyboard-shortcuts' => '01fca1f0',
-    'javelin-behavior-phabricator-line-linker' => 'c479ac01',
+    'javelin-behavior-phabricator-line-linker' => 'a9b946f8',
     'javelin-behavior-phabricator-nav' => '836f966d',
     'javelin-behavior-phabricator-notification-example' => '8ce821c5',
     'javelin-behavior-phabricator-object-selector' => '77c1f0b0',
@@ -1743,6 +1743,12 @@ return array(
       'javelin-uri',
       'phabricator-keyboard-shortcut',
     ),
+    'a9b946f8' => array(
+      'javelin-behavior',
+      'javelin-stratcom',
+      'javelin-dom',
+      'javelin-history',
+    ),
     'a9f88de2' => array(
       'javelin-behavior',
       'javelin-dom',
@@ -1931,12 +1937,6 @@ return array(
       'javelin-stratcom',
       'phabricator-tooltip',
     ),
-    'c479ac01' => array(
-      'javelin-behavior',
-      'javelin-stratcom',
-      'javelin-dom',
-      'javelin-history',
-    ),
     'c587b80f' => array(
       'javelin-install',
     ),
diff --git a/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
index 4fa826b690..cc8da38da9 100644
--- a/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
+++ b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
@@ -369,8 +369,9 @@ final class HarbormasterBuildLogRenderController
           'a',
           array(
             'href' => $uri.'$'.$display_line,
+            'data-n' => $display_line,
           ),
-          $display_line);
+          '');
 
         $line_cell = phutil_tag('th', array(), $display_line);
         $text_cell = phutil_tag('td', array(), $display_text);
diff --git a/webroot/rsrc/css/application/harbormaster/harbormaster.css b/webroot/rsrc/css/application/harbormaster/harbormaster.css
index 0e8574464d..38a65bf9a0 100644
--- a/webroot/rsrc/css/application/harbormaster/harbormaster.css
+++ b/webroot/rsrc/css/application/harbormaster/harbormaster.css
@@ -48,6 +48,12 @@
   user-select: none;
 }
 
+.harbormaster-log-table > tbody > tr > th a::before {
+  /* Render the line numbers into the document using a pseudo-element so that
+     the text is not copied. */
+  content: attr(data-n);
+}
+
 .harbormaster-log-table > tbody > tr > th a {
   display: block;
   color: {$darkbluetext};
diff --git a/webroot/rsrc/js/core/behavior-line-linker.js b/webroot/rsrc/js/core/behavior-line-linker.js
index ed93e6e11b..a0f99fb3bc 100644
--- a/webroot/rsrc/js/core/behavior-line-linker.js
+++ b/webroot/rsrc/js/core/behavior-line-linker.js
@@ -21,6 +21,16 @@ JX.behavior('phabricator-line-linker', function() {
 
   function getRowNumber(tr) {
     var th = tr.firstChild;
+
+    // If the "<th />" tag contains an "<a />" with "data-n" that we're using
+    // to prevent copy/paste of line numbers, use that.
+    if (th.firstChild) {
+      var line = th.firstChild.getAttribute('data-n');
+      if (line) {
+        return line;
+      }
+    }
+
     return +(th.textContent || th.innerText);
   }
 

From 1f40e50f7e50f7217b931f350e58a7572809c913 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Thu, 1 Mar 2018 10:43:28 -0800
Subject: [PATCH 36/39] Improve live Harbormaster log follow behaviors

Summary:
Depends on D19166. Ref T13088. When the user scrolls away from a followed log, break the focus lock.

Let users stop following a live log.

Show when lines are added more clearly.

Don't refresh quite as quickly give users a better shot at clicking the stop button.

These behaviors can probably be refined but are at least more plausible and less actively user-hostile than the first version of this behavior was.

Test Plan: Used `write-log --rate` to write a large log slowly. Clicked "Follow Log", followed for a bit. Scrolled away, still got live updates but no more scroll lock. Clicked stop, no more updates.

Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam

Maniphest Tasks: T13088

Differential Revision: https://secure.phabricator.com/D19167
---
 resources/celerity/map.php                    | 14 +++---
 .../HarbormasterBuildLogRenderController.php  | 26 +++++++++--
 .../application/harbormaster/harbormaster.css | 28 ++++++++++++
 .../harbormaster/behavior-harbormaster-log.js | 43 +++++++++++++++----
 4 files changed, 93 insertions(+), 18 deletions(-)

diff --git a/resources/celerity/map.php b/resources/celerity/map.php
index 3ffa4bb058..f601b5cec3 100644
--- a/resources/celerity/map.php
+++ b/resources/celerity/map.php
@@ -78,7 +78,7 @@ return array(
     'rsrc/css/application/feed/feed.css' => 'ecd4ec57',
     'rsrc/css/application/files/global-drag-and-drop.css' => 'b556a948',
     'rsrc/css/application/flag/flag.css' => 'bba8f811',
-    'rsrc/css/application/harbormaster/harbormaster.css' => 'cd73d427',
+    'rsrc/css/application/harbormaster/harbormaster.css' => '730a4a3c',
     'rsrc/css/application/herald/herald-test.css' => 'a52e323e',
     'rsrc/css/application/herald/herald.css' => 'cd8d0134',
     'rsrc/css/application/maniphest/report.css' => '9b9580b7',
@@ -416,7 +416,7 @@ return array(
     'rsrc/js/application/drydock/drydock-live-operation-status.js' => '901935ef',
     'rsrc/js/application/files/behavior-icon-composer.js' => '8499b6ab',
     'rsrc/js/application/files/behavior-launch-icon-composer.js' => '48086888',
-    'rsrc/js/application/harbormaster/behavior-harbormaster-log.js' => 'ab1173d1',
+    'rsrc/js/application/harbormaster/behavior-harbormaster-log.js' => '191b4909',
     'rsrc/js/application/herald/HeraldRuleEditor.js' => 'dca75c0e',
     'rsrc/js/application/herald/PathTypeahead.js' => 'f7fc67ec',
     'rsrc/js/application/herald/herald-rule-editor.js' => '7ebaeed3',
@@ -579,7 +579,7 @@ return array(
     'font-fontawesome' => 'e838e088',
     'font-lato' => 'c7ccd872',
     'global-drag-and-drop-css' => 'b556a948',
-    'harbormaster-css' => 'cd73d427',
+    'harbormaster-css' => '730a4a3c',
     'herald-css' => 'cd8d0134',
     'herald-rule-editor' => 'dca75c0e',
     'herald-test-css' => 'a52e323e',
@@ -636,7 +636,7 @@ return array(
     'javelin-behavior-event-all-day' => 'b41537c9',
     'javelin-behavior-fancy-datepicker' => 'ecf4e799',
     'javelin-behavior-global-drag-and-drop' => '960f6a39',
-    'javelin-behavior-harbormaster-log' => 'ab1173d1',
+    'javelin-behavior-harbormaster-log' => '191b4909',
     'javelin-behavior-herald-rule-editor' => '7ebaeed3',
     'javelin-behavior-high-security-warning' => 'a464fe03',
     'javelin-behavior-history-install' => '7ee2b591',
@@ -1022,6 +1022,9 @@ return array(
     '185bbd53' => array(
       'javelin-install',
     ),
+    '191b4909' => array(
+      'javelin-behavior',
+    ),
     '1ad0a787' => array(
       'javelin-install',
       'javelin-reactor',
@@ -1765,9 +1768,6 @@ return array(
       'javelin-util',
       'phabricator-prefab',
     ),
-    'ab1173d1' => array(
-      'javelin-behavior',
-    ),
     'ab2f381b' => array(
       'javelin-request',
       'javelin-behavior',
diff --git a/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
index cc8da38da9..7a5050becf 100644
--- a/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
+++ b/src/applications/harbormaster/controller/HarbormasterBuildLogRenderController.php
@@ -720,12 +720,16 @@ final class HarbormasterBuildLogRenderController
 
   private function renderLiveRow($log_size) {
     $icon_down = id(new PHUIIconView())
-      ->setIcon('fa-chevron-down');
+      ->setIcon('fa-angle-double-down');
+
+    $icon_pause = id(new PHUIIconView())
+      ->setIcon('fa-pause');
 
     $follow = javelin_tag(
       'a',
       array(
         'sigil' => 'harbormaster-log-expand harbormaster-log-live',
+        'class' => 'harbormaster-log-follow-start',
         'meta' => array(
           'headOffset' => $log_size,
           'head' => 0,
@@ -737,8 +741,21 @@ final class HarbormasterBuildLogRenderController
         $icon_down,
         ' ',
         pht('Follow Log'),
+      ));
+
+    $stop_following = javelin_tag(
+      'a',
+      array(
+        'sigil' => 'harbormaster-log-expand',
+        'class' => 'harbormaster-log-follow-stop',
+        'meta' => array(
+          'stop' => true,
+        ),
+      ),
+      array(
+        $icon_pause,
         ' ',
-        $icon_down,
+        pht('Stop Following Log'),
       ));
 
     $expand_cells = array(
@@ -747,7 +764,10 @@ final class HarbormasterBuildLogRenderController
         array(
           'class' => 'harbormaster-log-follow',
         ),
-        $follow),
+        array(
+          $follow,
+          $stop_following,
+        )),
     );
 
     return $this->renderActionTable($expand_cells);
diff --git a/webroot/rsrc/css/application/harbormaster/harbormaster.css b/webroot/rsrc/css/application/harbormaster/harbormaster.css
index 38a65bf9a0..f3b75a397b 100644
--- a/webroot/rsrc/css/application/harbormaster/harbormaster.css
+++ b/webroot/rsrc/css/application/harbormaster/harbormaster.css
@@ -139,3 +139,31 @@
 .harbormaster-log-expand-down .phui-icon-view {
   margin: 0 4px 0 0;
 }
+
+
+.harbormaster-log-following .harbormaster-log-table
+  .harbormaster-log-follow-start {
+  display: none;
+}
+
+.harbormaster-log-table .harbormaster-log-follow-stop {
+  display: none;
+}
+
+.harbormaster-log-following .harbormaster-log-table
+  .harbormaster-log-follow-stop {
+  display: block;
+}
+
+.harbormaster-log-appear > td {
+  animation: harbormaster-fade-in 1s linear;
+}
+
+@keyframes harbormaster-fade-in {
+  0% {
+    opacity: 0.5;
+  }
+  100% {
+    opacity: 1.0;
+  }
+}
diff --git a/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js b/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js
index 2b91106725..5bff2c1bb0 100644
--- a/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js
+++ b/webroot/rsrc/js/application/harbormaster/behavior-harbormaster-log.js
@@ -5,7 +5,9 @@
 
 JX.behavior('harbormaster-log', function(config) {
   var contentNode = JX.$(config.contentNodeID);
+
   var following = false;
+  var autoscroll = false;
 
   JX.DOM.listen(contentNode, 'click', 'harbormaster-log-expand', function(e) {
     if (!e.isNormalClick()) {
@@ -14,20 +16,29 @@ JX.behavior('harbormaster-log', function(config) {
 
     e.kill();
 
-    expand(e.getTarget());
+    expand(e.getTarget(), true);
   });
 
-  function expand(node) {
+  function expand(node, is_action) {
     var row = JX.DOM.findAbove(node, 'tr');
     row = JX.DOM.findAbove(row, 'tr');
 
     var data = JX.Stratcom.getData(node);
 
+    if (data.stop) {
+      following = false;
+      autoscroll = false;
+      JX.DOM.alterClass(contentNode, 'harbormaster-log-following', false);
+      return;
+    }
+
     var uri = new JX.URI(config.renderURI)
       .addQueryParams(data);
 
-    if (data.live) {
+    if (data.live && is_action) {
       following = true;
+      autoscroll = true;
+      JX.DOM.alterClass(contentNode, 'harbormaster-log-following', true);
     }
 
     var request = new JX.Request(uri, function(r) {
@@ -46,18 +57,34 @@ JX.behavior('harbormaster-log', function(config) {
       if (data.live) {
         // If this was a live follow, scroll the new data into view. This is
         // probably intensely annoying in practice but seems cool for now.
-        var last_row = rows[rows.length - 1];
-        var tail_pos = JX.$V(last_row).y + JX.Vector.getDim(last_row).y;
-        var view_y = JX.Vector.getViewport().y;
-        JX.DOM.scrollToPosition(null, (tail_pos - view_y) + 32);
+        if (autoscroll) {
+          var last_row = rows[rows.length - 1];
+          var tail_pos = JX.$V(last_row).y + JX.Vector.getDim(last_row).y;
+          var view_y = JX.Vector.getViewport().y;
+          JX.DOM.scrollToPosition(null, (tail_pos - view_y) + 32);
 
-        setTimeout(follow, 500);
+          // This will fire a scroll event, but we want to keep autoscroll
+          // enabled until we see an explicit scroll event by the user.
+          setTimeout(function() { autoscroll = true; }, 0);
+        }
+
+        setTimeout(follow, 2000);
+
+        for (var ii = 1; ii < (rows.length - 1); ii++) {
+          JX.DOM.alterClass(rows[ii], 'harbormaster-log-appear', true);
+        }
       }
     });
 
     request.send();
   }
 
+  // If the user explicitly scrolls while following a log, keep live updating
+  // it but stop following it with the scrollbar.
+  JX.Stratcom.listen('scroll', null, function() {
+    autoscroll = false;
+  });
+
   function follow() {
     if (!following) {
       return;

From 14fe941c349cafba70dda0662a2161ad5043e12d Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Thu, 1 Mar 2018 16:28:41 -0800
Subject: [PATCH 37/39] Reduce the cost of generating default user profile
 images

Summary:
See PHI413. You can pre-generate these with `bin/people profileimage --all`, but they're needlessly expensive to generate.

Streamline the workflow and cache some of the cacheable parts to reduce the generation cost.

Test Plan:
  - Ran `bin/people profileimage --all` and saw cost drop from {nav 15.801s > 4.839s}.
  - Set `defaultProfileImagePHID` to `NULL` in `phabricator_user.user` and purged caches with `bin/cache purge --all`.
  - Loaded user directory.
  - Saw default images regenerate relatively quickly.

Differential Revision: https://secure.phabricator.com/D19168
---
 src/__phutil_library_map__.php                |   2 -
 .../files/PhabricatorImageTransformer.php     |   6 +-
 ...abricatorFilesComposeAvatarBuiltinFile.php | 172 ++++++++++--------
 .../PhabricatorUserProfileImageCacheType.php  |  19 +-
 .../PhabricatorPeopleProfileImageWorkflow.php |   8 +-
 .../people/storage/PhabricatorUser.php        |   6 +-
 .../PhabricatorFilesComposeAvatarExample.php  |  95 ----------
 7 files changed, 111 insertions(+), 197 deletions(-)
 delete mode 100644 src/applications/uiexample/examples/PhabricatorFilesComposeAvatarExample.php

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
index 76bc79498e..bb08685634 100644
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -3044,7 +3044,6 @@ phutil_register_library_map(array(
     'PhabricatorFilesApplicationStorageEnginePanel' => 'applications/files/applicationpanel/PhabricatorFilesApplicationStorageEnginePanel.php',
     'PhabricatorFilesBuiltinFile' => 'applications/files/builtin/PhabricatorFilesBuiltinFile.php',
     'PhabricatorFilesComposeAvatarBuiltinFile' => 'applications/files/builtin/PhabricatorFilesComposeAvatarBuiltinFile.php',
-    'PhabricatorFilesComposeAvatarExample' => 'applications/uiexample/examples/PhabricatorFilesComposeAvatarExample.php',
     'PhabricatorFilesComposeIconBuiltinFile' => 'applications/files/builtin/PhabricatorFilesComposeIconBuiltinFile.php',
     'PhabricatorFilesConfigOptions' => 'applications/files/config/PhabricatorFilesConfigOptions.php',
     'PhabricatorFilesManagementCatWorkflow' => 'applications/files/management/PhabricatorFilesManagementCatWorkflow.php',
@@ -8628,7 +8627,6 @@ phutil_register_library_map(array(
     'PhabricatorFilesApplicationStorageEnginePanel' => 'PhabricatorApplicationConfigurationPanel',
     'PhabricatorFilesBuiltinFile' => 'Phobject',
     'PhabricatorFilesComposeAvatarBuiltinFile' => 'PhabricatorFilesBuiltinFile',
-    'PhabricatorFilesComposeAvatarExample' => 'PhabricatorUIExample',
     'PhabricatorFilesComposeIconBuiltinFile' => 'PhabricatorFilesBuiltinFile',
     'PhabricatorFilesConfigOptions' => 'PhabricatorApplicationConfigOptions',
     'PhabricatorFilesManagementCatWorkflow' => 'PhabricatorFilesManagementWorkflow',
diff --git a/src/applications/files/PhabricatorImageTransformer.php b/src/applications/files/PhabricatorImageTransformer.php
index df4243cfdd..f29ce97835 100644
--- a/src/applications/files/PhabricatorImageTransformer.php
+++ b/src/applications/files/PhabricatorImageTransformer.php
@@ -333,8 +333,12 @@ final class PhabricatorImageTransformer extends Phobject {
       return null;
     }
 
+    // NOTE: Empirically, the highest compression level (9) seems to take
+    // up to twice as long as the default compression level (6) but produce
+    // only slightly smaller files (10% on avatars, 3% on screenshots).
+
     ob_start();
-    $result = imagepng($image, null, 9);
+    $result = imagepng($image, null, 6);
     $output = ob_get_clean();
 
     if (!$result) {
diff --git a/src/applications/files/builtin/PhabricatorFilesComposeAvatarBuiltinFile.php b/src/applications/files/builtin/PhabricatorFilesComposeAvatarBuiltinFile.php
index fcd6f97601..4f827f7603 100644
--- a/src/applications/files/builtin/PhabricatorFilesComposeAvatarBuiltinFile.php
+++ b/src/applications/files/builtin/PhabricatorFilesComposeAvatarBuiltinFile.php
@@ -7,8 +7,79 @@ final class PhabricatorFilesComposeAvatarBuiltinFile
   private $color;
   private $border;
 
+  private $maps = array();
+
   const VERSION = 'v1';
 
+  public function updateUser(PhabricatorUser $user) {
+    $username = $user->getUsername();
+
+    $image_map = $this->getMap('image');
+    $initial = phutil_utf8_strtoupper(substr($username, 0, 1));
+    $pack = $this->pickMap('pack', $username);
+    $icon = "alphanumeric/{$pack}/{$initial}.png";
+    if (!isset($image_map[$icon])) {
+      $icon = "alphanumeric/{$pack}/_default.png";
+    }
+
+    $border = $this->pickMap('border', $username);
+    $color = $this->pickMap('color', $username);
+
+    $data = $this->composeImage($color, $icon, $border);
+    $name = $this->getImageDisplayName($color, $icon, $border);
+
+    $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
+
+      $file = PhabricatorFile::newFromFileData(
+        $data,
+        array(
+          'name' => $name,
+          'profile' => true,
+          'canCDN' => true,
+        ));
+
+      $user
+        ->setDefaultProfileImagePHID($file->getPHID())
+        ->setDefaultProfileImageVersion(self::VERSION)
+        ->saveWithoutIndex();
+
+    unset($unguarded);
+
+    return $file;
+  }
+
+  private function getMap($map_key) {
+    if (!isset($this->maps[$map_key])) {
+      switch ($map_key) {
+        case 'pack':
+          $map = $this->newPackMap();
+          break;
+        case 'image':
+          $map = $this->newImageMap();
+          break;
+        case 'color':
+          $map = $this->newColorMap();
+          break;
+        case 'border':
+          $map = $this->newBorderMap();
+          break;
+        default:
+          throw new Exception(pht('Unknown map "%s".', $map_key));
+      }
+      $this->maps[$map_key] = $map;
+    }
+
+    return $this->maps[$map_key];
+  }
+
+  private function pickMap($map_key, $username) {
+    $map = $this->getMap($map_key);
+    $seed = $username.'_'.$map_key;
+    $key = PhabricatorHash::digestToRange($seed, 0, count($map) - 1);
+    return $map[$key];
+  }
+
+
   public function setIcon($icon) {
     $this->icon = $icon;
     return $this;
@@ -46,15 +117,22 @@ final class PhabricatorFilesComposeAvatarBuiltinFile
   }
 
   public function getBuiltinDisplayName() {
-    $icon = $this->getIcon();
-    $color = $this->getColor();
-    $border = implode(',', $this->getBorder());
+    return $this->getImageDisplayName(
+      $this->getIcon(),
+      $this->getColor(),
+      $this->getBorder());
+  }
+
+  private function getImageDisplayName($icon, $color, $border) {
+    $border = implode(',', $border);
     return "{$icon}-{$color}-{$border}.png";
   }
 
   public function loadBuiltinFileData() {
     return $this->composeImage(
-      $this->getColor(), $this->getIcon(), $this->getBorder());
+      $this->getColor(),
+      $this->getIcon(),
+      $this->getBorder());
   }
 
   private function composeImage($color, $image, $border) {
@@ -68,7 +146,7 @@ final class PhabricatorFilesComposeAvatarBuiltinFile
 
     $color_const = hexdec(trim($color, '#'));
     $true_border = self::rgba2gd($border);
-    $image_map = self::getImageMap();
+    $image_map = $this->getMap('image');
     $data = Filesystem::readFile($image_map[$image]);
 
     $img = imagecreatefromstring($data);
@@ -111,10 +189,10 @@ final class PhabricatorFilesComposeAvatarBuiltinFile
     $b = $rgba[2];
     $a = $rgba[3];
     $a = (1 - $a) * 255;
-      return ($a << 24) | ($r << 16) | ($g << 8) | $b;
+    return ($a << 24) | ($r << 16) | ($g << 8) | $b;
   }
 
-  public static function getImageMap() {
+  private function newImageMap() {
     $root = dirname(phutil_get_library_root('phabricator'));
     $root = $root.'/resources/builtin/alphanumeric/';
 
@@ -131,64 +209,7 @@ final class PhabricatorFilesComposeAvatarBuiltinFile
     return $map;
   }
 
-  public function getUniqueProfileImage($username) {
-    $pack_map = $this->getImagePackMap();
-    $image_map = $this->getImageMap();
-    $color_map = $this->getColorMap();
-    $border_map = $this->getBorderMap();
-    $file = phutil_utf8_strtoupper(substr($username, 0, 1));
-
-    $pack_count = count($pack_map);
-    $color_count = count($color_map);
-    $border_count = count($border_map);
-
-    $pack_seed = $username.'_pack';
-    $color_seed = $username.'_color';
-    $border_seed = $username.'_border';
-
-    $pack_key =
-      PhabricatorHash::digestToRange($pack_seed, 0, $pack_count - 1);
-    $color_key =
-      PhabricatorHash::digestToRange($color_seed, 0, $color_count - 1);
-    $border_key =
-      PhabricatorHash::digestToRange($border_seed, 0, $border_count - 1);
-
-    $pack = $pack_map[$pack_key];
-    $icon = 'alphanumeric/'.$pack.'/'.$file.'.png';
-    $color = $color_map[$color_key];
-    $border = $border_map[$border_key];
-
-    if (!isset($image_map[$icon])) {
-    $icon = 'alphanumeric/'.$pack.'/_default.png';
-    }
-
-    return array('color' => $color, 'icon' => $icon, 'border' => $border);
-  }
-
-  public function getUserProfileImageFile($username) {
-    $unique = $this->getUniqueProfileImage($username);
-
-    $composer = id(new self())
-      ->setIcon($unique['icon'])
-      ->setColor($unique['color'])
-      ->setBorder($unique['border']);
-
-    $data = $composer->loadBuiltinFileData();
-
-    $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
-    $file = PhabricatorFile::newFromFileData(
-      $data,
-      array(
-        'name' => $composer->getBuiltinDisplayName(),
-        'profile' => true,
-        'canCDN' => true,
-      ));
-    unset($unguarded);
-
-    return $file;
-  }
-
-  public static function getImagePackMap() {
+  private function newPackMap() {
     $root = dirname(phutil_get_library_root('phabricator'));
     $root = $root.'/resources/builtin/alphanumeric/';
 
@@ -196,28 +217,24 @@ final class PhabricatorFilesComposeAvatarBuiltinFile
       ->withType('d')
       ->withFollowSymlinks(false)
       ->find();
+    $map = array_values($map);
 
-    return array_values($map);
+    return $map;
   }
 
-  public static function getBorderMap() {
-
-    $map = array(
+  private function newBorderMap() {
+    return array(
       array(0, 0, 0, 0),
       array(0, 0, 0, 0.3),
       array(255, 255, 255, 0.4),
       array(255, 255, 255, 0.7),
     );
-
-    return $map;
   }
 
-  public static function getColorMap() {
-    //
-    // Generated Colors
-    // http://tools.medialab.sciences-po.fr/iwanthue/
-    //
-    $map = array(
+  private function newColorMap() {
+    // Via: http://tools.medialab.sciences-po.fr/iwanthue/
+
+    return array(
       '#335862',
       '#2d5192',
       '#3c5da0',
@@ -447,7 +464,6 @@ final class PhabricatorFilesComposeAvatarBuiltinFile
       '#335862',
       '#335862',
     );
-    return $map;
   }
 
 }
diff --git a/src/applications/people/cache/PhabricatorUserProfileImageCacheType.php b/src/applications/people/cache/PhabricatorUserProfileImageCacheType.php
index e2a535cdd4..8babff859f 100644
--- a/src/applications/people/cache/PhabricatorUserProfileImageCacheType.php
+++ b/src/applications/people/cache/PhabricatorUserProfileImageCacheType.php
@@ -45,21 +45,10 @@ final class PhabricatorUserProfileImageCacheType
       $generate_users[] = $user;
     }
 
-    // Generate Files for anyone without a default
-    foreach ($generate_users as $generate_user) {
-      $generate_user_phid = $generate_user->getPHID();
-      $generate_username = $generate_user->getUsername();
-      $generate_version = PhabricatorFilesComposeAvatarBuiltinFile::VERSION;
-      $generate_file = id(new PhabricatorFilesComposeAvatarBuiltinFile())
-        ->getUserProfileImageFile($generate_username);
-
-      $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
-      $generate_user->setDefaultProfileImagePHID($generate_file->getPHID());
-      $generate_user->setDefaultProfileImageVersion($generate_version);
-      $generate_user->save();
-      unset($unguarded);
-
-      $file_phids[$generate_user_phid] = $generate_file->getPHID();
+    $generator = new PhabricatorFilesComposeAvatarBuiltinFile();
+    foreach ($generate_users as $user) {
+      $file = $generator->updateUser($user);
+      $file_phids[$user->getPHID()] = $file->getPHID();
     }
 
     if ($file_phids) {
diff --git a/src/applications/people/management/PhabricatorPeopleProfileImageWorkflow.php b/src/applications/people/management/PhabricatorPeopleProfileImageWorkflow.php
index 358756cecd..8bf3c8e118 100644
--- a/src/applications/people/management/PhabricatorPeopleProfileImageWorkflow.php
+++ b/src/applications/people/management/PhabricatorPeopleProfileImageWorkflow.php
@@ -51,6 +51,7 @@ final class PhabricatorPeopleProfileImageWorkflow
     }
 
     $version = PhabricatorFilesComposeAvatarBuiltinFile::VERSION;
+    $generator = new PhabricatorFilesComposeAvatarBuiltinFile();
 
     foreach ($iterator as $user) {
       $username = $user->getUsername();
@@ -63,16 +64,13 @@ final class PhabricatorPeopleProfileImageWorkflow
       }
 
       if ($default_phid == null || $is_force || $generate) {
-        $file = id(new PhabricatorFilesComposeAvatarBuiltinFile())
-          ->getUserProfileImageFile($username);
-        $user->setDefaultProfileImagePHID($file->getPHID());
-        $user->setDefaultProfileImageVersion($version);
-        $user->save();
         $console->writeOut(
           "%s\n",
           pht(
             'Generating profile image for "%s".',
             $username));
+
+        $generator->updateUser($user);
       } else {
         $console->writeOut(
           "%s\n",
diff --git a/src/applications/people/storage/PhabricatorUser.php b/src/applications/people/storage/PhabricatorUser.php
index 6c8b7ac6b5..ba8b1cf292 100644
--- a/src/applications/people/storage/PhabricatorUser.php
+++ b/src/applications/people/storage/PhabricatorUser.php
@@ -267,6 +267,10 @@ final class PhabricatorUser
     return !($this->getPHID() === null);
   }
 
+  public function saveWithoutIndex() {
+    return parent::save();
+  }
+
   public function save() {
     if (!$this->getConduitCertificate()) {
       $this->setConduitCertificate($this->generateConduitCertificate());
@@ -276,7 +280,7 @@ final class PhabricatorUser
       $this->setAccountSecret(Filesystem::readRandomCharacters(64));
     }
 
-    $result = parent::save();
+    $result = $this->saveWithoutIndex();
 
     if ($this->profile) {
       $this->profile->save();
diff --git a/src/applications/uiexample/examples/PhabricatorFilesComposeAvatarExample.php b/src/applications/uiexample/examples/PhabricatorFilesComposeAvatarExample.php
deleted file mode 100644
index 57a65863db..0000000000
--- a/src/applications/uiexample/examples/PhabricatorFilesComposeAvatarExample.php
+++ /dev/null
@@ -1,95 +0,0 @@
-<?php
-
-final class PhabricatorFilesComposeAvatarExample extends PhabricatorUIExample {
-
-  public function getName() {
-    return pht('Avatars');
-  }
-
-  public function getDescription() {
-    return pht('Tests various color palettes and sizes.');
-  }
-
-  public function getCategory() {
-    return pht('Technical');
-  }
-
-  public function renderExample() {
-    $request = $this->getRequest();
-    $viewer = $request->getUser();
-
-    $colors = PhabricatorFilesComposeAvatarBuiltinFile::getColorMap();
-    $packs = PhabricatorFilesComposeAvatarBuiltinFile::getImagePackMap();
-    $builtins = PhabricatorFilesComposeAvatarBuiltinFile::getImageMap();
-    $borders = PhabricatorFilesComposeAvatarBuiltinFile::getBorderMap();
-
-    $images = array();
-    foreach ($builtins as $builtin => $raw_file) {
-      $file = PhabricatorFile::loadBuiltin($viewer, $builtin);
-      $images[] = $file->getBestURI();
-    }
-
-    $content = array();
-    shuffle($colors);
-    foreach ($colors as $color) {
-      shuffle($borders);
-      $color_const = hexdec(trim($color, '#'));
-      $border = head($borders);
-      $border_color = implode(', ', $border);
-
-      $styles = array();
-      $styles[] = 'background-color: '.$color.';';
-      $styles[] = 'display: inline-block;';
-      $styles[] = 'height: 42px;';
-      $styles[] = 'width: 42px;';
-      $styles[] = 'border-radius: 3px;';
-      $styles[] = 'border: 4px solid rgba('.$border_color.');';
-
-      shuffle($images);
-      $png = head($images);
-
-      $image = phutil_tag(
-        'img',
-        array(
-          'src' => $png,
-          'height' => 42,
-          'width' => 42,
-        ));
-
-      $tag = phutil_tag(
-        'div',
-        array(
-          'style' => implode(' ', $styles),
-        ),
-        $image);
-
-      $content[] = phutil_tag(
-        'div',
-        array(
-          'class' => 'mlr mlb',
-          'style' => 'float: left;',
-        ),
-        $tag);
-    }
-
-    $count = new PhutilNumber(
-      count($colors) * count($builtins) * count($borders));
-
-    $infoview = id(new PHUIInfoView())
-      ->setSeverity(PHUIInfoView::SEVERITY_NOTICE)
-      ->appendChild(pht('This installation can generate %s unique '.
-      'avatars. You can add additional image packs in '.
-      'resources/builtins/alphanumeric/.', $count));
-
-    $info = phutil_tag_div('pmb', $infoview);
-    $view = phutil_tag_div('ml', $content);
-
-    return phutil_tag(
-      'div',
-        array(),
-        array(
-          $info,
-          $view,
-        ));
-      }
-}

From 2121f2dea6979586f0edf4a4e2f0861f4d6e62f2 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Thu, 1 Mar 2018 18:38:26 -0800
Subject: [PATCH 38/39] Don't require project edit permission to create a
 project with members other than yourself

Summary: See PHI193. Previously, see similar D18763. Skip this legacy-style policy check when creating a project, since we know you can add members, even if the policy doesn't actually resolve in your favor.

Test Plan:
  - Created a project with edit policy "Members of project" and myself, plus any other user (so the code goes down this path, not the "join/leave" path) as members.

Differential Revision: https://secure.phabricator.com/D19169
---
 .../editor/PhabricatorProjectTransactionEditor.php  | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/applications/project/editor/PhabricatorProjectTransactionEditor.php b/src/applications/project/editor/PhabricatorProjectTransactionEditor.php
index de61c2a09b..2a90d6ce29 100644
--- a/src/applications/project/editor/PhabricatorProjectTransactionEditor.php
+++ b/src/applications/project/editor/PhabricatorProjectTransactionEditor.php
@@ -156,11 +156,14 @@ final class PhabricatorProjectTransactionEditor
                   PhabricatorPolicyCapability::CAN_EDIT);
               }
             } else {
-              // You need CAN_EDIT to change members other than yourself.
-              PhabricatorPolicyFilter::requireCapability(
-                $this->requireActor(),
-                $object,
-                PhabricatorPolicyCapability::CAN_EDIT);
+              if (!$this->getIsNewObject()) {
+                // You need CAN_EDIT to change members other than yourself.
+                // (PHI193) Just skip this check if we're creating a project.
+                PhabricatorPolicyFilter::requireCapability(
+                  $this->requireActor(),
+                  $object,
+                  PhabricatorPolicyCapability::CAN_EDIT);
+              }
             }
             return;
         }

From 42e5b8a04bece0abe4e018deb381347ac1bf7d37 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Fri, 2 Mar 2018 06:59:55 -0800
Subject: [PATCH 39/39] Include the primary domain in the
 Content-Security-Policy explicitly if there's no CDN

Summary:
Ref T4340. If you don't configure a CDN and visit a custom site (like a Phame blog site, or a CORGI sandbox internally) we serve resources from the main site. This violates the Content-Security-Policy.

When there's no CDN, include the primary domain in the CSP explicitly.

Test Plan: Loaded `local.www.phacility.com`, got resources.

Maniphest Tasks: T4340

Differential Revision: https://secure.phabricator.com/D19170
---
 src/aphront/response/AphrontResponse.php | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/aphront/response/AphrontResponse.php b/src/aphront/response/AphrontResponse.php
index 73c4e68f00..892417fcb1 100644
--- a/src/aphront/response/AphrontResponse.php
+++ b/src/aphront/response/AphrontResponse.php
@@ -109,7 +109,14 @@ abstract class AphrontResponse extends Phobject {
     if ($cdn) {
       $default = $this->newContentSecurityPolicySource($cdn);
     } else {
-      $default = "'self'";
+      // If an alternate file domain is not configured and the user is viewing
+      // a Phame blog on a custom domain or some other custom site, we'll still
+      // serve resources from the main site. Include the main site explicitly.
+
+      $base_uri = PhabricatorEnv::getURI('/');
+      $base_uri = $this->newContentSecurityPolicySource($base_uri);
+
+      $default = "'self' {$base_uri}";
     }
 
     $csp[] = "default-src {$default}";