Migrate Calendar away from stored-epoch fields

Summary:
Ref T10747. This deprecates "dateFrom", "dateTo", "allDayDateFrom", "allDayDateTo", and "recurrenceEndDate".

They are replaced with "utc*Epoch" fields (for querying) and CalendarDateTime objects (for start, end, until). These objects can represent the full range of dates and times expressible in ICS format, allowing us to import a wider range of ICS events.

Test Plan:
Ran migrations, viewed/edited Calendar, didn't catch anything catastrophcially broken.

This likely needs some followups, I'll keep it local for a bit until I'm confident I didn't break anything too catastrophically. I'm retaining the old data for now so we can likely fix things if it turns out there is some sort of issue.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T10747

Differential Revision: https://secure.phabricator.com/D16664

resources/sql/autopatches/20160715.event.03.allday.php

@@ -1,52 +1,3 @@
 <?php
 
-$table = new PhabricatorCalendarEvent();
+// This migration was replaced by "20161004.cal.01.noepoch.php".
-$conn = $table->establishConnection('w');
-
-// Previously, "All Day" events were stored with a start and end date set to
-// the earliest possible start and end seconds for the corresponding days. We
-// now store all day events with their "date" epochs as UTC, separate from
-// individual event times.
-$zone_min = new DateTimeZone('Pacific/Midway');
-$zone_max = new DateTimeZone('Pacific/Kiritimati');
-$zone_utc = new DateTimeZone('UTC');
-
-foreach (new LiskMigrationIterator($table) as $event) {
-  // If this event has already migrated, skip it.
-  if ($event->getAllDayDateFrom()) {
-    continue;
-  }
-
-  $is_all_day = $event->getIsAllDay();
-
-  $epoch_min = $event->getDateFrom();
-  $epoch_max = $event->getDateTo();
-
-  $date_min = new DateTime('@'.$epoch_min);
-  $date_max = new DateTime('@'.$epoch_max);
-
-  if ($is_all_day) {
-    $date_min->setTimeZone($zone_min);
-    $date_min->modify('+2 days');
-    $date_max->setTimeZone($zone_max);
-    $date_max->modify('-2 days');
-  } else {
-    $date_min->setTimeZone($zone_utc);
-    $date_max->setTimeZone($zone_utc);
-  }
-
-  $string_min = $date_min->format('Y-m-d');
-  $string_max = $date_max->format('Y-m-d 23:59:00');
-
-  $allday_min = id(new DateTime($string_min, $zone_utc))->format('U');
-  $allday_max = id(new DateTime($string_max, $zone_utc))->format('U');
-
-  queryfx(
-    $conn,
-    'UPDATE %T SET allDayDateFrom = %d, allDayDateTo = %d
-      WHERE id = %d',
-    $table->getTableName(),
-    $allday_min,
-    $allday_max,
-    $event->getID());
-}

resources/sql/autopatches/20161004.cal.01.noepoch.php

@@ -0,0 +1,125 @@
+<?php
+
+$table = new PhabricatorCalendarEvent();
+$conn = $table->establishConnection('w');
+$table_name = 'calendar_event';
+
+// Long ago, "All Day" events were stored with a start and end date set to
+// the earliest possible start and end seconds for the corresponding days. We
+// then moved to store all day events with their "date" epochs as UTC, separate
+// from individual event times. Both systems were later replaced with use of
+// CalendarDateTime.
+$zone_min = new DateTimeZone('Pacific/Midway');
+$zone_max = new DateTimeZone('Pacific/Kiritimati');
+$zone_utc = new DateTimeZone('UTC');
+
+foreach (new LiskRawMigrationIterator($conn, $table_name) as $row) {
+  $parameters = phutil_json_decode($row['parameters']);
+  if (isset($parameters['startDateTime'])) {
+    // This event has already been migrated.
+    continue;
+  }
+
+  $is_all_day = $row['isAllDay'];
+
+  if (empty($row['allDayDateFrom'])) {
+    // No "allDayDateFrom" means this is an old event which was never migrated
+    // by the earlier "20160715.event.03.allday.php" migration. The dateFrom
+    // and dateTo will be minimum and maximum earthly seconds for the event. We
+    // convert them to UTC if they were in extreme timezones.
+    $epoch_min = $row['dateFrom'];
+    $epoch_max = $row['dateTo'];
+
+    if ($is_all_day) {
+      $date_min = new DateTime('@'.$epoch_min);
+      $date_max = new DateTime('@'.$epoch_max);
+
+      $date_min->setTimeZone($zone_min);
+      $date_min->modify('+2 days');
+      $date_max->setTimeZone($zone_max);
+      $date_max->modify('-2 days');
+
+      $string_min = $date_min->format('Y-m-d');
+      $string_max = $date_max->format('Y-m-d 23:59:00');
+
+      $utc_min = id(new DateTime($string_min, $zone_utc))->format('U');
+      $utc_max = id(new DateTime($string_max, $zone_utc))->format('U');
+    } else {
+      $utc_min = $epoch_min;
+      $utc_max = $epoch_max;
+    }
+  } else {
+    // This is an event which was migrated already. We can pick the correct
+    // epoch timestamps based on the "isAllDay" flag.
+    if ($is_all_day) {
+      $utc_min = $row['allDayDateFrom'];
+      $utc_max = $row['allDayDateTo'];
+    } else {
+      $utc_min = $row['dateFrom'];
+      $utc_max = $row['dateTo'];
+    }
+  }
+
+  $utc_until = $row['recurrenceEndDate'];
+
+  $start_datetime = PhutilCalendarAbsoluteDateTime::newFromEpoch($utc_min);
+  if ($is_all_day) {
+    $start_datetime->setIsAllDay(true);
+  }
+
+  $end_datetime = PhutilCalendarAbsoluteDateTime::newFromEpoch($utc_max);
+  if ($is_all_day) {
+    $end_datetime->setIsAllDay(true);
+  }
+
+  if ($utc_until) {
+    $until_datetime = PhutilCalendarAbsoluteDateTime::newFromEpoch($utc_until);
+  } else {
+    $until_datetime = null;
+  }
+
+  $parameters['startDateTime'] = $start_datetime->toDictionary();
+  $parameters['endDateTime'] = $end_datetime->toDictionary();
+  if ($until_datetime) {
+    $parameters['untilDateTime'] = $until_datetime->toDictionary();
+  }
+
+  queryfx(
+    $conn,
+    'UPDATE %T SET parameters = %s WHERE id = %d',
+    $table_name,
+    phutil_json_encode($parameters),
+    $row['id']);
+}
+
+// Generate UTC epochs for all events. We can't readily do this one at a
+// time because instance UTC epochs rely on having the parent event.
+$viewer = PhabricatorUser::getOmnipotentUser();
+
+$all_events = id(new PhabricatorCalendarEventQuery())
+  ->setViewer($viewer)
+  ->execute();
+foreach ($all_events as $event) {
+  if ($event->getUTCInitialEpoch()) {
+    // Already migrated.
+    continue;
+  }
+
+  try {
+    $event->updateUTCEpochs();
+  } catch (Exception $ex) {
+    continue;
+  }
+
+  queryfx(
+    $conn,
+    'UPDATE %T SET
+      utcInitialEpoch = %d,
+      utcUntilEpoch = %nd,
+      utcInstanceEpoch = %nd WHERE id = %d',
+    $table_name,
+    $event->getUTCInitialEpoch(),
+    $event->getUTCUntilEpoch(),
+    $event->getUTCInstanceEpoch(),
+    $event->getID());
+}

src/applications/calendar/query/PhabricatorCalendarEventQuery.php

@@ -352,14 +352,14 @@ final class PhabricatorCalendarEventQuery
     if ($this->rangeBegin) {
       $where[] = qsprintf(
         $conn,
-        'event.dateTo >= %d OR event.isRecurring = 1',
+        '(event.utcUntilEpoch >= %d) OR (event.utcUntilEpoch IS NULL)',
         $this->rangeBegin - phutil_units('16 hours in seconds'));
     }
 
     if ($this->rangeEnd) {
       $where[] = qsprintf(
         $conn,
-        'event.dateFrom <= %d',
+        'event.utcInitialEpoch <= %d',
         $this->rangeEnd + phutil_units('16 hours in seconds'));
     }
 

src/applications/calendar/storage/PhabricatorCalendarEvent.php

@@ -17,10 +17,6 @@ final class PhabricatorCalendarEvent extends PhabricatorCalendarDAO
 
   protected $name;
   protected $hostPHID;
-  protected $dateFrom;
-  protected $dateTo;
-  protected $allDayDateFrom;
-  protected $allDayDateTo;
   protected $description;
   protected $isCancelled;
   protected $isAllDay;
@@ -30,7 +26,6 @@ final class PhabricatorCalendarEvent extends PhabricatorCalendarDAO
 
   protected $isRecurring = 0;
   protected $recurrenceFrequency = array();
-  protected $recurrenceEndDate;
 
   private $isGhostEvent = false;
   protected $instanceOfEventPHID;
@@ -51,6 +46,13 @@ final class PhabricatorCalendarEvent extends PhabricatorCalendarDAO
 
   private $viewerTimezone;
 
+  // TODO: DEPRECATED. Remove once we're sure the migrations worked.
+  protected $allDayDateFrom;
+  protected $allDayDateTo;
+  protected $dateFrom;
+  protected $dateTo;
+  protected $recurrenceEndDate;
+
   // Frequency Constants
   const FREQUENCY_DAILY = 'daily';
   const FREQUENCY_WEEKLY = 'weekly';
@@ -70,20 +72,6 @@ final class PhabricatorCalendarEvent extends PhabricatorCalendarDAO
 
     $now = PhabricatorTime::getNow();
 
-    $start = new DateTime('@'.$now);
-    $start->setTimeZone($actor->getTimeZone());
-
-    $start->setTime($start->format('H'), 0, 0);
-    $start->modify('+1 hour');
-    $end = id(clone $start)->modify('+1 hour');
-
-    $epoch_min = $start->format('U');
-    $epoch_max = $end->format('U');
-
-    $now_date = new DateTime('@'.$now);
-    $now_min = id(clone $now_date)->setTime(0, 0)->format('U');
-    $now_max = id(clone $now_date)->setTime(23, 59)->format('U');
-
     $default_icon = 'fa-calendar';
 
     $datetime_start = PhutilCalendarAbsoluteDateTime::newFromEpoch(
@@ -106,10 +94,10 @@ final class PhabricatorCalendarEvent extends PhabricatorCalendarDAO
       ->setEditPolicy($edit_policy)
       ->setSpacePHID($actor->getDefaultSpacePHID())
       ->attachInvitees(array())
-      ->setDateFrom($epoch_min)
+      ->setDateFrom(0)
-      ->setDateTo($epoch_max)
+      ->setDateTo(0)
-      ->setAllDayDateFrom($now_min)
+      ->setAllDayDateFrom(0)
-      ->setAllDayDateTo($now_max)
+      ->setAllDayDateTo(0)
       ->setStartDateTime($datetime_start)
       ->setEndDateTime($datetime_end)
       ->applyViewerTimezone($actor);
@@ -130,7 +118,11 @@ final class PhabricatorCalendarEvent extends PhabricatorCalendarDAO
       ->setSequenceIndex($sequence)
       ->setIsRecurring(true)
       ->setRecurrenceFrequency($this->getRecurrenceFrequency())
-      ->attachParentEvent($this);
+      ->attachParentEvent($this)
+      ->setAllDayDateFrom(0)
+      ->setAllDayDateTo(0)
+      ->setDateFrom(0)
+      ->setDateTo(0);
 
     return $child->copyFromParent($actor);
   }
@@ -187,11 +179,16 @@ final class PhabricatorCalendarEvent extends PhabricatorCalendarDAO
     $duration = $parent->getDuration();
     $epochs = $parent->getSequenceIndexEpochs($actor, $sequence, $duration);
 
+    $start_datetime = PhutilCalendarAbsoluteDateTime::newFromEpoch(
+      $epochs['dateFrom'],
+      $parent->newStartDateTime()->getTimezone());
+    $end_datetime = PhutilCalendarAbsoluteDateTime::newFromEpoch(
+      $epochs['dateTo'],
+      $parent->newEndDateTime()->getTimezone());
+
     $this
-      ->setDateFrom($epochs['dateFrom'])
+      ->setStartDateTime($start_datetime)
-      ->setDateTo($epochs['dateTo'])
+      ->setEndDateTime($end_datetime);
-      ->setAllDayDateFrom($epochs['allDayDateFrom'])
-      ->setAllDayDateTo($epochs['allDayDateTo']);
 
     return $this;
   }
@@ -213,12 +210,14 @@ final class PhabricatorCalendarEvent extends PhabricatorCalendarDAO
     $frequency = $this->getFrequencyUnit();
     $modify_key = '+'.$sequence.' '.$frequency;
 
-    $date = $this->getDateFrom();
+    $date_time = $this->newStartDateTime()
-    $date_time = PhabricatorTime::getDateTimeFromEpoch($date, $viewer);
+      ->setViewerTimezone($viewer->getTimezoneIdentifier())
+      ->newPHPDateTime();
+
     $date_time->modify($modify_key);
     $date = $date_time->format('U');
 
-    $end_date = $this->getRecurrenceEndDate();
+    $end_date = $this->getUntilDateTimeEpoch();
     if ($end_date && $date > $end_date) {
       throw new Exception(
         pht(
@@ -227,21 +226,9 @@ final class PhabricatorCalendarEvent extends PhabricatorCalendarDAO
           $sequence));
     }
 
-    $utc = new DateTimeZone('UTC');
-
-    $allday_from = $this->getAllDayDateFrom();
-    $allday_date = new DateTime('@'.$allday_from, $utc);
-    $allday_date->setTimeZone($utc);
-    $allday_date->modify($modify_key);
-
-    $allday_min = $allday_date->format('U');
-    $allday_duration = ($this->getAllDayDateTo() - $allday_from);
-
     return array(
       'dateFrom' => $date,
       'dateTo' => $date + $duration,
-      'allDayDateFrom' => $allday_min,
-      'allDayDateTo' => $allday_min + $allday_duration,
     );
   }
 
@@ -280,24 +267,6 @@ final class PhabricatorCalendarEvent extends PhabricatorCalendarDAO
     return ($this->getEndDateTimeEpoch() - $this->getStartDateTimeEpoch());
   }
 
-  public function getDateEpochForTimezone(
-    $epoch,
-    $src_zone,
-    $format,
-    $adjust,
-    $dst_zone) {
-
-    $src = new DateTime('@'.$epoch);
-    $src->setTimeZone($src_zone);
-
-    if (strlen($adjust)) {
-      $adjust = ' '.$adjust;
-    }
-
-    $dst = new DateTime($src->format($format).$adjust, $dst_zone);
-    return $dst->format('U');
-  }
-
   public function updateUTCEpochs() {
     // The "intitial" epoch is the start time of the event, in UTC.
     $start_date = $this->newStartDateTime()
@@ -314,9 +283,9 @@ final class PhabricatorCalendarEvent extends PhabricatorCalendarDAO
       $until_epoch = $end_date->getEpoch();
     } else {
       $until_epoch = null;
-      $until_date = $this->newUntilDateTime()
+      $until_date = $this->newUntilDateTime();
-        ->setViewerTimezone('UTC');
       if ($until_date) {
+        $until_date->setViewerTimezone('UTC');
         $duration = $this->newDuration();
         $until_epoch = id(new PhutilCalendarRelativeDateTime())
           ->setOrigin($until_date)
@@ -377,23 +346,25 @@ final class PhabricatorCalendarEvent extends PhabricatorCalendarDAO
       self::CONFIG_AUX_PHID => true,
       self::CONFIG_COLUMN_SCHEMA => array(
         'name' => 'text',
-        'dateFrom' => 'epoch',
-        'dateTo' => 'epoch',
-        'allDayDateFrom' => 'epoch',
-        'allDayDateTo' => 'epoch',
         'description' => 'text',
         'isCancelled' => 'bool',
         'isAllDay' => 'bool',
         'icon' => 'text32',
         'mailKey' => 'bytes20',
         'isRecurring' => 'bool',
-        'recurrenceEndDate' => 'epoch?',
         'instanceOfEventPHID' => 'phid?',
         'sequenceIndex' => 'uint32?',
         'isStub' => 'bool',
         'utcInitialEpoch' => 'epoch',
         'utcUntilEpoch' => 'epoch?',
         'utcInstanceEpoch' => 'epoch?',
+
+        // TODO: DEPRECATED.
+        'allDayDateFrom' => 'epoch',
+        'allDayDateTo' => 'epoch',
+        'dateFrom' => 'epoch',
+        'dateTo' => 'epoch',
+        'recurrenceEndDate' => 'epoch?',
       ),
       self::CONFIG_KEY_SCHEMA => array(
         'key_date' => array(
@@ -814,7 +785,7 @@ final class PhabricatorCalendarEvent extends PhabricatorCalendarDAO
       return null;
     }
 
-    $epochs = $this->getParent()->getSequenceIndexEpochs(
+    $epochs = $this->getParentEvent()->getSequenceIndexEpochs(
       new PhabricatorUser(),
       $this->getSequenceIndex(),
       $this->getDuration());

src/applications/calendar/xaction/PhabricatorCalendarEventEndDateTransaction.php

@@ -13,16 +13,6 @@ final class PhabricatorCalendarEventEndDateTransaction
   public function applyInternalEffects($object, $value) {
     $actor = $this->getActor();
 
-    // TODO: DEPRECATED.
-    $object->setDateTo($value);
-    $object->setAllDayDateTo(
-      $object->getDateEpochForTimezone(
-        $value,
-        $actor->getTimeZone(),
-        'Y-m-d 23:59:00',
-        null,
-        new DateTimeZone('UTC')));
-
     $datetime = PhutilCalendarAbsoluteDateTime::newFromEpoch(
       $value,
       $actor->getTimezoneIdentifier());

src/applications/calendar/xaction/PhabricatorCalendarEventStartDateTransaction.php

@@ -13,16 +13,6 @@ final class PhabricatorCalendarEventStartDateTransaction
   public function applyInternalEffects($object, $value) {
     $actor = $this->getActor();
 
-    // TODO: DEPRECATED.
-    $object->setDateFrom($value);
-    $object->setAllDayDateFrom(
-      $object->getDateEpochForTimezone(
-        $value,
-        $actor->getTimeZone(),
-        'Y-m-d',
-        null,
-        new DateTimeZone('UTC')));
-
     $datetime = PhutilCalendarAbsoluteDateTime::newFromEpoch(
       $value,
       $actor->getTimezoneIdentifier());