improvements to patch process
- improve messaging and add instructions how to fix the patch - fix application of patch stack. Now base commit and order of patches should be correct - set origin to llvm-project fork to avoid accidental operations on origin - use annotations to make issues visible - create commits with original author of change - add patches we tried to apply to artifacts - patching / commits works locally if --push-branch=false is set - misc renames / wordings
This commit is contained in:
Mikhail Goncharov
parent
1ce0e1ff35
commit
90e7224ba9
4 changed files with 384 additions and 326 deletions
|
|
@ -18,7 +18,7 @@
|
||||||
|
|
||||||
set -uo pipefail
|
set -uo pipefail
|
||||||
|
|
||||||
scripts/phabtalk/apply_patch2.py $ph_buildable_diff \
|
scripts/patch_diff.py $ph_buildable_diff \
|
||||||
--path "${BUILDKITE_BUILD_PATH}"/llvm-project-fork \
|
--path "${BUILDKITE_BUILD_PATH}"/llvm-project-fork \
|
||||||
--token $CONDUIT_TOKEN \
|
--token $CONDUIT_TOKEN \
|
||||||
--url $PHABRICATOR_HOST \
|
--url $PHABRICATOR_HOST \
|
||||||
|
|
@ -30,7 +30,7 @@ scripts/phabtalk/apply_patch2.py $ph_buildable_diff \
|
||||||
EXIT_STATUS=$?
|
EXIT_STATUS=$?
|
||||||
|
|
||||||
if [ $EXIT_STATUS -ne 0 ]; then
|
if [ $EXIT_STATUS -ne 0 ]; then
|
||||||
scripts/add_phabricator_artifact.py --phid="$ph_target_phid" --url="$BUILDKITE_BUILD_URL" --name="Buildkite apply patch"
|
scripts/add_phabricator_artifact.py --phid="$ph_target_phid" --url="$BUILDKITE_BUILD_URL" --name="patch application failed"
|
||||||
scripts/set_build_status.py
|
scripts/set_build_status.py
|
||||||
echo failed
|
echo failed
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
|
|
@ -2,11 +2,16 @@ import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import urllib.parse
|
||||||
|
import shlex
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
import backoff
|
import backoff
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
context_style = {}
|
||||||
|
styles = ['default', 'info', 'success', 'warning', 'error']
|
||||||
|
|
||||||
|
|
||||||
def upload_file(base_dir: str, file: str):
|
def upload_file(base_dir: str, file: str):
|
||||||
"""
|
"""
|
||||||
|
|
@ -25,6 +30,34 @@ def upload_file(base_dir: str, file: str):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def annotate(message: str, style: str = 'default', context: str = 'default', append: bool = True):
|
||||||
|
"""
|
||||||
|
Adds an annotation for that currently running build.
|
||||||
|
Note that last `style` applied to the same `context` takes precedence.
|
||||||
|
"""
|
||||||
|
if style not in styles:
|
||||||
|
style = 'default'
|
||||||
|
# Pick most severe style so far.
|
||||||
|
context_style.setdefault(context, 0)
|
||||||
|
context_style[context] = max(styles.index(style), context_style[context])
|
||||||
|
style = styles[context_style[context]]
|
||||||
|
if append:
|
||||||
|
message += '\n\n'
|
||||||
|
r = subprocess.run(f"buildkite-agent annotate {shlex.quote(message)}"
|
||||||
|
f' --style={shlex.quote(style)}'
|
||||||
|
f" {'--append' if append else ''}"
|
||||||
|
f" --context={shlex.quote(context)}", shell=True, capture_output=True)
|
||||||
|
logging.debug(f'annotate call {r}')
|
||||||
|
if r.returncode != 0:
|
||||||
|
logging.warning(message)
|
||||||
|
|
||||||
|
|
||||||
|
def feedback_url():
|
||||||
|
title = f"buildkite build {os.getenv('BUILDKITE_PIPELINE_SLUG')} {os.getenv('BUILDKITE_BUILD_NUMBER')}"
|
||||||
|
return f'https://github.com/google/llvm-premerge-checks/issues/new?assignees=&labels=bug' \
|
||||||
|
f'&template=bug_report.md&title={urllib.parse.quote(title)}'
|
||||||
|
|
||||||
|
|
||||||
class BuildkiteApi:
|
class BuildkiteApi:
|
||||||
def __init__(self, token: str, organization: str):
|
def __init__(self, token: str, organization: str):
|
||||||
self.token = token
|
self.token = token
|
||||||
|
|
@ -37,7 +70,7 @@ class BuildkiteApi:
|
||||||
url = f'https://api.buildkite.com/v2/organizations/{self.organization}/pipelines/{pipeline}/builds/{build_number}'
|
url = f'https://api.buildkite.com/v2/organizations/{self.organization}/pipelines/{pipeline}/builds/{build_number}'
|
||||||
response = requests.get(url, headers={'Authorization': authorization})
|
response = requests.get(url, headers={'Authorization': authorization})
|
||||||
if response.status_code != 200:
|
if response.status_code != 200:
|
||||||
raise Exception(f'Builkite responded with non-OK status: {re.status_code}')
|
raise Exception(f'Builkite responded with non-OK status: {response.status_code}')
|
||||||
return response.json()
|
return response.json()
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
348
scripts/patch_diff.py
Executable file
348
scripts/patch_diff.py
Executable file
|
|
@ -0,0 +1,348 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# Copyright 2019 Google LLC
|
||||||
|
#
|
||||||
|
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# https://llvm.org/LICENSE.txt
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import datetime
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from typing import List, Optional, Tuple, Dict
|
||||||
|
|
||||||
|
import backoff
|
||||||
|
from buildkite_utils import annotate, feedback_url, upload_file
|
||||||
|
import git
|
||||||
|
from phabricator import Phabricator
|
||||||
|
|
||||||
|
"""URL of upstream LLVM repository."""
|
||||||
|
LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project'
|
||||||
|
FORK_REMOTE_URL = 'ssh://git@github.com/llvm-premerge-tests/llvm-project'
|
||||||
|
|
||||||
|
"""How far back the script searches in the git history to find Revisions that
|
||||||
|
have already landed. """
|
||||||
|
APPLIED_SCAN_LIMIT = datetime.timedelta(days=90)
|
||||||
|
|
||||||
|
|
||||||
|
class ApplyPatch:
|
||||||
|
"""Apply a diff from Phabricator on local working copy.
|
||||||
|
|
||||||
|
This script is a rewrite of `arc patch` to accommodate for dependencies
|
||||||
|
that have already landed, but could not be identified by `arc patch`.
|
||||||
|
|
||||||
|
For a given diff_id, this class will get the dependencies listed on Phabricator.
|
||||||
|
For each dependency D it will check the diff history:
|
||||||
|
- if D has already landed, skip it.
|
||||||
|
- If D has not landed, it will download the patch for D and try to apply it locally.
|
||||||
|
Once this class has applied all dependencies, it will apply the original diff.
|
||||||
|
|
||||||
|
This script must be called from the root folder of a local checkout of
|
||||||
|
https://github.com/llvm/llvm-project or given a path to clone into.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, path: str, diff_id: int, token: str, url: str, git_hash: str,
|
||||||
|
phid: str, push_branch: bool = False):
|
||||||
|
self.push_branch = push_branch # type: bool
|
||||||
|
self.conduit_token = token # type: Optional[str]
|
||||||
|
self.host = url # type: Optional[str]
|
||||||
|
self.diff_id = diff_id # type: int
|
||||||
|
self.phid = phid # type: str
|
||||||
|
if not self.host.endswith('/api/'):
|
||||||
|
self.host += '/api/'
|
||||||
|
self.phab = self.create_phab()
|
||||||
|
self.base_revision = git_hash # type: str
|
||||||
|
self.branch_base_hexsha = ''
|
||||||
|
self.apply_diff_counter = 0
|
||||||
|
self.build_dir = os.getcwd()
|
||||||
|
self.revision_id = ''
|
||||||
|
|
||||||
|
if not os.path.isdir(path):
|
||||||
|
logging.info(f'{path} does not exist, cloning repository...')
|
||||||
|
self.repo = git.Repo.clone_from(FORK_REMOTE_URL, path)
|
||||||
|
else:
|
||||||
|
logging.info('repository exist, will reuse')
|
||||||
|
self.repo = git.Repo(path) # type: git.Repo
|
||||||
|
self.repo.remote('origin').set_url(FORK_REMOTE_URL)
|
||||||
|
os.chdir(path)
|
||||||
|
logging.info(f'working dir {os.getcwd()}')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def branch_name(self):
|
||||||
|
"""Name used for the git branch."""
|
||||||
|
return f'phab-diff-{self.diff_id}'
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
"""try to apply the patch from phabricator
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
diff = self.get_diff(self.diff_id)
|
||||||
|
revision = self.get_revision(diff.revisionID)
|
||||||
|
url = f"https://reviews.llvm.org/D{revision['id']}?id={diff['id']}"
|
||||||
|
annotate(f"Patching changes [{url}]({url})", style='info', context='patch_diff')
|
||||||
|
self.reset_repository()
|
||||||
|
self.revision_id = revision['id']
|
||||||
|
dependencies = self.get_dependencies(revision)
|
||||||
|
dependencies.reverse() # Now revisions will be from oldest to newest.
|
||||||
|
missing, landed = self.classify_revisions(dependencies)
|
||||||
|
if len(dependencies) > 0:
|
||||||
|
logging.info('This diff depends on: {}'.format(revision_list_to_str(dependencies)))
|
||||||
|
logging.info(' Already landed: {}'.format(revision_list_to_str(landed)))
|
||||||
|
logging.info(' Will be applied: {}'.format(revision_list_to_str(missing)))
|
||||||
|
plan = []
|
||||||
|
for r in missing:
|
||||||
|
d = self.get_diff(r['diffs'][0])
|
||||||
|
plan.append((r, d))
|
||||||
|
plan.append((revision, diff))
|
||||||
|
logging.info('Planning to apply in order:')
|
||||||
|
for (r, d) in plan:
|
||||||
|
logging.info(f"https://reviews.llvm.org/D{r['id']}?id={d['id']}")
|
||||||
|
# Pick the newest known commit as a base for patches.
|
||||||
|
base_commit = None
|
||||||
|
for (r, d) in plan:
|
||||||
|
c = self.find_commit(d['sourceControlBaseRevision'])
|
||||||
|
if c is None:
|
||||||
|
logging.warning(f"D{r['id']}#{d['id']} commit {d['sourceControlBaseRevision']} does not exist")
|
||||||
|
continue
|
||||||
|
if base_commit is None:
|
||||||
|
logging.info(f"D{r['id']}#{d['id']} commit {c.hexsha} exists")
|
||||||
|
base_commit = c
|
||||||
|
elif c.committed_datetime > base_commit.committed_datetime:
|
||||||
|
logging.info(f"D{r['id']}#{d['id']} commit {c.hexsha} has a later commit date then"
|
||||||
|
f"{base_commit.hexsha}")
|
||||||
|
base_commit = c
|
||||||
|
if self.base_revision != 'auto':
|
||||||
|
logging.info(f'Base revision "{self.base_revision}" is set by command argument. Will use '
|
||||||
|
f'instead of resolved "{base_commit}"')
|
||||||
|
base_commit = self.find_commit(self.base_revision)
|
||||||
|
if base_commit is None:
|
||||||
|
base_commit = self.repo.heads['master'].commit
|
||||||
|
annotate(f"Cannot find a base git revision. Will use current HEAD.",
|
||||||
|
style='warning', context='patch_diff')
|
||||||
|
self.create_branch(base_commit)
|
||||||
|
for (r, d) in plan:
|
||||||
|
if not self.apply_diff(d, r):
|
||||||
|
return 1
|
||||||
|
if self.push_branch:
|
||||||
|
self.repo.git.push('--force', 'origin', self.branch_name)
|
||||||
|
annotate(f"Created branch [{self.branch_name}]"
|
||||||
|
f"(https://github.com/llvm-premerge-tests/llvm-project/tree/{self.branch_name}).\n\n"
|
||||||
|
f"To checkout locally, run in your copy of llvm-project directory:\n\n"
|
||||||
|
"```shell\n"
|
||||||
|
"git remote add premerge git@github.com:llvm-premerge-tests/llvm-project.git #first time\n"
|
||||||
|
f"git fetch premerge {self.branch_name}\n"
|
||||||
|
f"git checkout -b {self.branch_name} --track premerge/{self.branch_name}\n"
|
||||||
|
"```",
|
||||||
|
style='success',
|
||||||
|
context='patch_diff')
|
||||||
|
logging.info('Branch {} has been pushed'.format(self.branch_name))
|
||||||
|
return 0
|
||||||
|
except Exception as e:
|
||||||
|
annotate(f":bk-status-failed: Unexpected error. Consider [creating a bug]({feedback_url()}).",
|
||||||
|
style='error', context='patch_diff')
|
||||||
|
logging.error(f'exception: {e}')
|
||||||
|
return 1
|
||||||
|
|
||||||
|
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
|
||||||
|
def reset_repository(self):
|
||||||
|
"""Update local git repo and origin.
|
||||||
|
|
||||||
|
As origin is disjoint from upstream, it needs to be updated by this script.
|
||||||
|
"""
|
||||||
|
logging.info('Syncing local, origin and upstream...')
|
||||||
|
self.repo.git.clean('-ffxdq')
|
||||||
|
self.repo.git.reset('--hard')
|
||||||
|
self.repo.git.fetch('--all')
|
||||||
|
self.repo.git.checkout('master')
|
||||||
|
if 'upstream' not in self.repo.remotes:
|
||||||
|
self.repo.create_remote('upstream', url=LLVM_GITHUB_URL)
|
||||||
|
self.repo.remotes.upstream.fetch()
|
||||||
|
self.repo.git.pull('origin', 'master')
|
||||||
|
self.repo.git.pull('upstream', 'master')
|
||||||
|
if self.push_branch:
|
||||||
|
self.repo.git.push('origin', 'master')
|
||||||
|
|
||||||
|
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
|
||||||
|
def find_commit(self, rev):
|
||||||
|
try:
|
||||||
|
return self.repo.commit(rev)
|
||||||
|
except ValueError as e:
|
||||||
|
return None
|
||||||
|
|
||||||
|
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
|
||||||
|
def create_branch(self, base_commit: git.Commit):
|
||||||
|
if self.branch_name in self.repo.heads:
|
||||||
|
self.repo.delete_head('--force', self.branch_name)
|
||||||
|
logging.info(f'creating branch {self.branch_name} at {base_commit.hexsha}')
|
||||||
|
new_branch = self.repo.create_head(self.branch_name, base_commit.hexsha)
|
||||||
|
self.repo.head.reference = new_branch
|
||||||
|
self.repo.head.reset(index=True, working_tree=True)
|
||||||
|
self.branch_base_hexsha = self.repo.head.commit.hexsha
|
||||||
|
logging.info('Base branch revision is {}'.format(self.repo.head.commit.hexsha))
|
||||||
|
annotate(f"Branch {self.branch_name} base revision is `{self.branch_base_hexsha}`.",
|
||||||
|
style='info', context='patch_diff')
|
||||||
|
|
||||||
|
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
|
||||||
|
def commit(self, revision: Dict, diff: Dict):
|
||||||
|
"""Commit the current state and annotates with the revision info."""
|
||||||
|
self.repo.git.add('-A')
|
||||||
|
diff.setdefault('authorName', 'unknown')
|
||||||
|
diff.setdefault('authorEmail', 'unknown')
|
||||||
|
author = git.Actor(name=diff['authorName'], email=diff['authorEmail'])
|
||||||
|
message = (f"{revision['title']}\n\n"
|
||||||
|
f"Automated commit created by applying diff {self.diff_id}\n"
|
||||||
|
f"\n"
|
||||||
|
f"Phabricator-ID: {self.phid}\n"
|
||||||
|
f"Review-ID: {diff_to_str(revision['id'])}\n")
|
||||||
|
self.repo.index.commit(message=message, author=author)
|
||||||
|
|
||||||
|
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
|
||||||
|
def create_phab(self):
|
||||||
|
phab = Phabricator(token=self.conduit_token, host=self.host)
|
||||||
|
phab.update_interfaces()
|
||||||
|
return phab
|
||||||
|
|
||||||
|
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
|
||||||
|
def get_diff(self, diff_id: int):
|
||||||
|
"""Get a diff from Phabricator based on it's diff id."""
|
||||||
|
return self.phab.differential.getdiff(diff_id=diff_id)
|
||||||
|
|
||||||
|
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
|
||||||
|
def get_revision(self, revision_id: int):
|
||||||
|
"""Get a revision from Phabricator based on its revision id."""
|
||||||
|
return self.phab.differential.query(ids=[revision_id])[0]
|
||||||
|
|
||||||
|
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
|
||||||
|
def get_revisions(self, *, phids: List[str] = None):
|
||||||
|
"""Get a list of revisions from Phabricator based on their PH-IDs."""
|
||||||
|
if phids is None:
|
||||||
|
raise Exception('_get_revisions phids is None')
|
||||||
|
if not phids:
|
||||||
|
# Handle an empty query locally. Otherwise the connection
|
||||||
|
# will time out.
|
||||||
|
return []
|
||||||
|
return self.phab.differential.query(phids=phids)
|
||||||
|
|
||||||
|
def get_dependencies(self, revision: Dict) -> List[Dict]:
|
||||||
|
"""Recursively resolves dependencies of the given revision.
|
||||||
|
They are listed in reverse chronological order - from most recent to least recent."""
|
||||||
|
dependency_ids = revision['auxiliary']['phabricator:depends-on']
|
||||||
|
revisions = self.get_revisions(phids=dependency_ids)
|
||||||
|
result = []
|
||||||
|
for r in revisions:
|
||||||
|
result.append(r)
|
||||||
|
sub = self.get_dependencies(r)
|
||||||
|
result.extend(sub)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def apply_diff(self, diff: Dict, revision: Dict) -> bool:
|
||||||
|
"""Download and apply a diff to the local working copy."""
|
||||||
|
logging.info(f"Applying {diff['id']} for revision {revision['id']}...")
|
||||||
|
patch = self.get_raw_diff(str(diff['id']))
|
||||||
|
self.apply_diff_counter += 1
|
||||||
|
patch_file = f"{self.apply_diff_counter}_{diff['id']}.patch"
|
||||||
|
with open(os.path.join(self.build_dir, patch_file), 'wt') as f:
|
||||||
|
f.write(patch)
|
||||||
|
# For annotate to properly link this file it must exist before the upload.
|
||||||
|
upload_file(self.build_dir, patch_file)
|
||||||
|
logging.debug(f'raw patch:\n{patch}')
|
||||||
|
proc = subprocess.run('git apply -', input=patch, shell=True, text=True,
|
||||||
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
|
if proc.returncode != 0:
|
||||||
|
message = f":bk-status-failed: Failed to apply [{patch_file}](artifact://{patch_file}).\n\n"
|
||||||
|
if self.revision_id != revision['id']:
|
||||||
|
message += f"**Attention! D{revision['id']} is one of the dependencies of the target " \
|
||||||
|
f"revision D{self.revision_id}.**\n\n"
|
||||||
|
message += (f"No testing is possible because we couldn't apply the patch.\n\n"
|
||||||
|
f"---\n\n"
|
||||||
|
'### Troubleshooting\n\n'
|
||||||
|
'More information is available in the log of of *create branch* step. '
|
||||||
|
f"All patches applied are available as *Artifacts*.\n\n"
|
||||||
|
f":bulb: The patch may not apply if it includes only the most recent of "
|
||||||
|
f"multiple local commits. Try to upload a patch with\n"
|
||||||
|
f"```shell\n"
|
||||||
|
f"arc diff `git merge-base HEAD origin` --update D{revision['id']}\n"
|
||||||
|
f"```\n\n"
|
||||||
|
f"to include all local changes.\n\n"
|
||||||
|
'---\n\n'
|
||||||
|
f"If this case could have been handled better, please [create a bug]({feedback_url()}).")
|
||||||
|
annotate(message,
|
||||||
|
style='error',
|
||||||
|
context='patch_diff')
|
||||||
|
return False
|
||||||
|
self.commit(revision, diff)
|
||||||
|
return True
|
||||||
|
|
||||||
|
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
|
||||||
|
def get_raw_diff(self, diff_id: str) -> str:
|
||||||
|
return self.phab.differential.getrawdiff(diffID=diff_id).response
|
||||||
|
|
||||||
|
def get_landed_revisions(self):
|
||||||
|
"""Get list of landed revisions from current git branch."""
|
||||||
|
diff_regex = re.compile(r'^Differential Revision: https://reviews\.llvm\.org/(.*)$', re.MULTILINE)
|
||||||
|
earliest_commit = None
|
||||||
|
rev = self.base_revision
|
||||||
|
age_limit = datetime.datetime.now() - APPLIED_SCAN_LIMIT
|
||||||
|
if rev == 'auto': # FIXME: use revison that created the branch
|
||||||
|
rev = 'master'
|
||||||
|
for commit in self.repo.iter_commits(rev):
|
||||||
|
if datetime.datetime.fromtimestamp(commit.committed_date) < age_limit:
|
||||||
|
break
|
||||||
|
earliest_commit = commit
|
||||||
|
result = diff_regex.search(commit.message)
|
||||||
|
if result is not None:
|
||||||
|
yield result.group(1)
|
||||||
|
if earliest_commit is not None:
|
||||||
|
logging.info(f'Earliest analyzed commit in history {earliest_commit.hexsha}, '
|
||||||
|
f'{earliest_commit.committed_datetime}')
|
||||||
|
return
|
||||||
|
|
||||||
|
def classify_revisions(self, revisions: List[Dict]) -> Tuple[List[Dict], List[Dict]]:
|
||||||
|
"""Check which of the dependencies have already landed on the current branch."""
|
||||||
|
landed_deps = []
|
||||||
|
missing_deps = []
|
||||||
|
for d in revisions:
|
||||||
|
if diff_to_str(d['id']) in self.get_landed_revisions():
|
||||||
|
landed_deps.append(d)
|
||||||
|
else:
|
||||||
|
missing_deps.append(d)
|
||||||
|
return missing_deps, landed_deps
|
||||||
|
|
||||||
|
|
||||||
|
def diff_to_str(diff: int) -> str:
|
||||||
|
"""Convert a diff id to a string with leading "D"."""
|
||||||
|
return 'D{}'.format(diff)
|
||||||
|
|
||||||
|
|
||||||
|
def revision_list_to_str(diffs: List[Dict]) -> str:
|
||||||
|
"""Convert list of diff ids to a comma separated list, prefixed with "D"."""
|
||||||
|
return ', '.join([diff_to_str(d['id']) for d in diffs])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser(description='Apply Phabricator patch to working directory.')
|
||||||
|
parser.add_argument('diff_id', type=int)
|
||||||
|
parser.add_argument('--path', type=str, help='repository path', default=os.getcwd())
|
||||||
|
parser.add_argument('--token', type=str, default=None, help='Conduit API token')
|
||||||
|
parser.add_argument('--url', type=str, default='https://reviews.llvm.org', help='Phabricator URL')
|
||||||
|
parser.add_argument('--commit', dest='commit', type=str, default='auto',
|
||||||
|
help='Use this commit as a base. For "auto" tool tries to pick the base commit itself')
|
||||||
|
parser.add_argument('--push-branch', action='store_true', dest='push_branch',
|
||||||
|
help='choose if branch shall be pushed to origin')
|
||||||
|
parser.add_argument('--phid', type=str, default=None, help='Phabricator ID of the review this commit pertains to')
|
||||||
|
parser.add_argument('--log-level', type=str, default='INFO')
|
||||||
|
args = parser.parse_args()
|
||||||
|
logging.basicConfig(level=args.log_level, format='%(levelname)-7s %(message)s')
|
||||||
|
patcher = ApplyPatch(args.path, args.diff_id, args.token, args.url, args.commit, args.phid, args.push_branch)
|
||||||
|
sys.exit(patcher.run())
|
||||||
|
|
@ -1,323 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
# Copyright 2019 Google LLC
|
|
||||||
#
|
|
||||||
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# https://llvm.org/LICENSE.txt
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import datetime
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import socket
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
from typing import List, Optional, Tuple
|
|
||||||
|
|
||||||
import backoff
|
|
||||||
from git import Repo, BadName, GitCommandError
|
|
||||||
from phabricator import Phabricator
|
|
||||||
|
|
||||||
# FIXME: maybe move to config file
|
|
||||||
"""URL of upstream LLVM repository."""
|
|
||||||
LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project'
|
|
||||||
FORK_REMOTE_URL = 'ssh://git@github.com/llvm-premerge-tests/llvm-project'
|
|
||||||
|
|
||||||
"""How far back the script searches in the git history to find Revisions that
|
|
||||||
have already landed. """
|
|
||||||
APPLIED_SCAN_LIMIT = datetime.timedelta(days=90)
|
|
||||||
|
|
||||||
|
|
||||||
class ApplyPatch:
|
|
||||||
"""Apply a diff from Phabricator on local working copy.
|
|
||||||
|
|
||||||
This script is a rewrite of `arc patch` to accomodate for dependencies
|
|
||||||
that have already landed, but could not be identified by `arc patch`.
|
|
||||||
|
|
||||||
For a given diff_id, this class will get the dependencies listed on Phabricator.
|
|
||||||
For each dependency D it will check the diff history:
|
|
||||||
- if D has already landed, skip it.
|
|
||||||
- If D has not landed, it will download the patch for D and try to apply it locally.
|
|
||||||
Once this class has applied all dependencies, it will apply the diff itself.
|
|
||||||
|
|
||||||
This script must be called from the root folder of a local checkout of
|
|
||||||
https://github.com/llvm/llvm-project or given a path to clone into.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, path: str, diff_id: int, token: str, url: str, git_hash: str,
|
|
||||||
phid: str, push_branch: bool = False):
|
|
||||||
self.push_branch = push_branch # type: bool
|
|
||||||
self.conduit_token = token # type: Optional[str]
|
|
||||||
self.host = url # type: Optional[str]
|
|
||||||
self._load_arcrc()
|
|
||||||
self.diff_id = diff_id # type: int
|
|
||||||
self.phid = phid # type: str
|
|
||||||
if not self.host.endswith('/api/'):
|
|
||||||
self.host += '/api/'
|
|
||||||
self.phab = self._create_phab()
|
|
||||||
self.base_revision = git_hash # type: str
|
|
||||||
|
|
||||||
if not os.path.isdir(path):
|
|
||||||
logging.info(f'{path} does not exist, cloning repository')
|
|
||||||
# TODO: progress of clonning
|
|
||||||
self.repo = Repo.clone_from(FORK_REMOTE_URL, path)
|
|
||||||
else:
|
|
||||||
logging.info('repository exist, will reuse')
|
|
||||||
self.repo = Repo(path) # type: Repo
|
|
||||||
# TODO: set origin url to fork
|
|
||||||
os.chdir(path)
|
|
||||||
logging.info(f'working dir {os.getcwd()}')
|
|
||||||
|
|
||||||
@property
|
|
||||||
def branch_name(self):
|
|
||||||
"""Name used for the git branch."""
|
|
||||||
return 'phab-diff-{}'.format(self.diff_id)
|
|
||||||
|
|
||||||
def _load_arcrc(self):
|
|
||||||
"""Load arc configuration from file if not set."""
|
|
||||||
if self.conduit_token is not None or self.host is not None:
|
|
||||||
return
|
|
||||||
logging.info('Loading configuration from ~/.arcrc file')
|
|
||||||
with open(os.path.expanduser('~/.arcrc'), 'r') as arcrc_file:
|
|
||||||
arcrc = json.load(arcrc_file)
|
|
||||||
# use the first host configured in the file
|
|
||||||
self.host = next(iter(arcrc['hosts']))
|
|
||||||
self.conduit_token = arcrc['hosts'][self.host]['token']
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
"""try to apply the patch from phabricator
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
try:
|
|
||||||
self._refresh_master()
|
|
||||||
revision_id, dependencies, base_revision = self._get_dependencies(self.diff_id)
|
|
||||||
dependencies.reverse() # Arrange deps in chronological order.
|
|
||||||
if self.base_revision != 'auto':
|
|
||||||
logging.info('Using base revision provided by command line\n{} instead of resolved\n{}'.format(
|
|
||||||
self.base_revision, base_revision))
|
|
||||||
base_revision = self.base_revision
|
|
||||||
self._create_branch(base_revision)
|
|
||||||
logging.info('git reset, git cleanup...')
|
|
||||||
self.repo.git.reset('--hard')
|
|
||||||
self.repo.git.clean('-ffxdq')
|
|
||||||
logging.info('Analyzing {}'.format(diff_to_str(revision_id)))
|
|
||||||
if len(dependencies) > 0:
|
|
||||||
logging.info('This diff depends on: {}'.format(diff_list_to_str(dependencies)))
|
|
||||||
missing, landed = self._get_missing_landed_dependencies(dependencies)
|
|
||||||
logging.info(' Already landed: {}'.format(diff_list_to_str(landed)))
|
|
||||||
logging.info(' Will be applied: {}'.format(diff_list_to_str(missing)))
|
|
||||||
if missing:
|
|
||||||
for revision in missing:
|
|
||||||
self._apply_revision(revision)
|
|
||||||
# FIXME: submit every Revision individually to get nicer history, use original user name
|
|
||||||
self.repo.config_writer().set_value("user", "name", "myusername").release()
|
|
||||||
self.repo.config_writer().set_value("user", "email", "myemail@example.com").release()
|
|
||||||
self.repo.git.commit('-a', '-m', 'dependencies')
|
|
||||||
logging.info('All depended diffs are applied')
|
|
||||||
logging.info('applying original diff')
|
|
||||||
self._apply_diff(self.diff_id, revision_id)
|
|
||||||
if self.push_branch:
|
|
||||||
self._commit_and_push(revision_id)
|
|
||||||
else:
|
|
||||||
self.repo.git.add('-u', '.')
|
|
||||||
return 0
|
|
||||||
except Exception as e:
|
|
||||||
logging.error(f'exception: {e}')
|
|
||||||
return 1
|
|
||||||
|
|
||||||
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
|
|
||||||
def _refresh_master(self):
|
|
||||||
"""Update local git repo and origin.
|
|
||||||
|
|
||||||
As origin is disjoint from upstream, it needs to be updated by this script.
|
|
||||||
"""
|
|
||||||
logging.info('Syncing local, origin and upstream...')
|
|
||||||
self.repo.git.clean('-ffxdq')
|
|
||||||
self.repo.git.reset('--hard')
|
|
||||||
self.repo.git.fetch('--all')
|
|
||||||
self.repo.git.checkout('master')
|
|
||||||
if 'upstream' not in self.repo.remotes:
|
|
||||||
self.repo.create_remote('upstream', url=LLVM_GITHUB_URL)
|
|
||||||
self.repo.remotes.upstream.fetch()
|
|
||||||
self.repo.git.pull('origin', 'master')
|
|
||||||
self.repo.git.pull('upstream', 'master')
|
|
||||||
if self.push_branch:
|
|
||||||
self.repo.git.push('origin', 'master')
|
|
||||||
|
|
||||||
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
|
|
||||||
def _create_branch(self, base_revision: Optional[str]):
|
|
||||||
self.repo.git.fetch('--all')
|
|
||||||
try:
|
|
||||||
if self.branch_name in self.repo.heads:
|
|
||||||
self.repo.delete_head('--force', self.branch_name)
|
|
||||||
except:
|
|
||||||
logging.warning('cannot delete branch {}'.format(self.branch_name))
|
|
||||||
try:
|
|
||||||
commit = self.repo.commit(base_revision)
|
|
||||||
except Exception as e:
|
|
||||||
logging.info('Cannot resolve revision {}: {}, going to use "master" instead.'.format(base_revision, e))
|
|
||||||
commit = self.repo.heads['master'].commit
|
|
||||||
logging.info(f'creating branch {self.branch_name} at {commit.hexsha}')
|
|
||||||
new_branch = self.repo.create_head(self.branch_name, commit.hexsha)
|
|
||||||
self.repo.head.reference = new_branch
|
|
||||||
self.repo.head.reset(index=True, working_tree=True)
|
|
||||||
logging.info('Base branch revision is {}'.format(self.repo.head.commit.hexsha))
|
|
||||||
|
|
||||||
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
|
|
||||||
def _commit_and_push(self, revision_id):
|
|
||||||
"""Commit the patch and push it to origin."""
|
|
||||||
if not self.push_branch:
|
|
||||||
return
|
|
||||||
|
|
||||||
self.repo.git.add('-A')
|
|
||||||
message = """
|
|
||||||
Applying diff {}
|
|
||||||
|
|
||||||
Phabricator-ID: {}
|
|
||||||
Review-ID: {}
|
|
||||||
""".format(self.diff_id, self.phid, diff_to_str(revision_id))
|
|
||||||
self.repo.index.commit(message=message)
|
|
||||||
self.repo.git.push('--force', 'origin', self.branch_name)
|
|
||||||
logging.info('Branch {} pushed to origin'.format(self.branch_name))
|
|
||||||
pass
|
|
||||||
|
|
||||||
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
|
|
||||||
def _create_phab(self):
|
|
||||||
phab = Phabricator(token=self.conduit_token, host=self.host)
|
|
||||||
phab.update_interfaces()
|
|
||||||
return phab
|
|
||||||
|
|
||||||
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
|
|
||||||
def _get_diff(self, diff_id: int):
|
|
||||||
"""Get a diff from Phabricator based on it's diff id."""
|
|
||||||
return self.phab.differential.getdiff(diff_id=diff_id)
|
|
||||||
|
|
||||||
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
|
|
||||||
def _get_revision(self, revision_id: int):
|
|
||||||
"""Get a revision from Phabricator based on its revision id."""
|
|
||||||
return self.phab.differential.query(ids=[revision_id])[0]
|
|
||||||
|
|
||||||
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
|
|
||||||
def _get_revisions(self, *, phids: List[str] = None):
|
|
||||||
"""Get a list of revisions from Phabricator based on their PH-IDs."""
|
|
||||||
if phids is None:
|
|
||||||
raise Exception('_get_revisions phids is None')
|
|
||||||
if not phids:
|
|
||||||
# Handle an empty query locally. Otherwise the connection
|
|
||||||
# will time out.
|
|
||||||
return []
|
|
||||||
return self.phab.differential.query(phids=phids)
|
|
||||||
|
|
||||||
def _get_dependencies(self, diff_id) -> Tuple[int, List[int], str]:
|
|
||||||
"""Get all dependencies for the diff.
|
|
||||||
They are listed in reverse chronological order - from most recent to least recent."""
|
|
||||||
|
|
||||||
logging.info('Getting dependencies of {}'.format(diff_id))
|
|
||||||
diff = self._get_diff(diff_id)
|
|
||||||
logging.debug(f'diff object: {diff}')
|
|
||||||
revision_id = int(diff.revisionID)
|
|
||||||
revision = self._get_revision(revision_id)
|
|
||||||
logging.debug(f'revision object: {revision}')
|
|
||||||
base_revision = diff['sourceControlBaseRevision']
|
|
||||||
if base_revision is None or len(base_revision) == 0:
|
|
||||||
base_revision = 'master'
|
|
||||||
dependency_ids = revision['auxiliary']['phabricator:depends-on']
|
|
||||||
revisions = self._get_revisions(phids=dependency_ids)
|
|
||||||
result = []
|
|
||||||
# Recursively resolve dependencies of those diffs.
|
|
||||||
for r in revisions:
|
|
||||||
_, sub, _ = self._get_dependencies(r['diffs'][0])
|
|
||||||
result.append(r['id'])
|
|
||||||
result.extend(sub)
|
|
||||||
|
|
||||||
return revision_id, result, base_revision
|
|
||||||
|
|
||||||
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
|
|
||||||
def _apply_diff(self, diff_id: int, revision_id: int):
|
|
||||||
"""Download and apply a diff to the local working copy."""
|
|
||||||
logging.info('Applying diff {} for revision {}...'.format(diff_id, diff_to_str(revision_id)))
|
|
||||||
diff = self.phab.differential.getrawdiff(diffID=str(diff_id)).response
|
|
||||||
logging.debug(f'diff {diff_id}:\n{diff}')
|
|
||||||
proc = subprocess.run('git apply -', input=diff, shell=True, text=True,
|
|
||||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
||||||
if proc.returncode != 0:
|
|
||||||
raise Exception('Applying patch failed:\n{}'.format(proc.stdout + proc.stderr))
|
|
||||||
|
|
||||||
def _apply_revision(self, revision_id: int):
|
|
||||||
"""Download and apply the latest diff of a revision to the local working copy."""
|
|
||||||
revision = self._get_revision(revision_id)
|
|
||||||
# take the diff_id with the highest number, this should be latest one
|
|
||||||
diff_id = max(revision['diffs'])
|
|
||||||
logging.info(f'picking diff from revision {revision_id}: {diff_id} from {revision["diffs"]}')
|
|
||||||
self._apply_diff(diff_id, revision_id)
|
|
||||||
|
|
||||||
def _get_landed_revisions(self):
|
|
||||||
"""Get list of landed revisions from current git branch."""
|
|
||||||
diff_regex = re.compile(r'^Differential Revision: https://reviews\.llvm\.org/(.*)$', re.MULTILINE)
|
|
||||||
earliest_commit = None
|
|
||||||
rev = self.base_revision
|
|
||||||
age_limit = datetime.datetime.now() - APPLIED_SCAN_LIMIT
|
|
||||||
if rev == 'auto': # FIXME: use revison that created the branch
|
|
||||||
rev = 'master'
|
|
||||||
for commit in self.repo.iter_commits(rev):
|
|
||||||
if datetime.datetime.fromtimestamp(commit.committed_date) < age_limit:
|
|
||||||
break
|
|
||||||
earliest_commit = commit
|
|
||||||
result = diff_regex.search(commit.message)
|
|
||||||
if result is not None:
|
|
||||||
yield result.group(1)
|
|
||||||
if earliest_commit is not None:
|
|
||||||
logging.info(f'Earliest analyzed commit in history {earliest_commit.hexsha}, '
|
|
||||||
f'{earliest_commit.committed_datetime}')
|
|
||||||
return
|
|
||||||
|
|
||||||
def _get_missing_landed_dependencies(self, dependencies: List[int]) -> Tuple[List[int], List[int]]:
|
|
||||||
"""Check which of the dependencies have already landed on the current branch."""
|
|
||||||
landed_deps = []
|
|
||||||
missing_deps = []
|
|
||||||
for dependency in dependencies:
|
|
||||||
if diff_to_str(dependency) in self._get_landed_revisions():
|
|
||||||
landed_deps.append(dependency)
|
|
||||||
else:
|
|
||||||
missing_deps.append(dependency)
|
|
||||||
return missing_deps, landed_deps
|
|
||||||
|
|
||||||
|
|
||||||
def diff_to_str(diff: int) -> str:
|
|
||||||
"""Convert a diff id to a string with leading "D"."""
|
|
||||||
return 'D{}'.format(diff)
|
|
||||||
|
|
||||||
|
|
||||||
def diff_list_to_str(diffs: List[int]) -> str:
|
|
||||||
"""Convert list of diff ids to a comma separated list, prefixed with "D"."""
|
|
||||||
return ', '.join([diff_to_str(d) for d in diffs])
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
parser = argparse.ArgumentParser(description='Apply Phabricator patch to working directory.')
|
|
||||||
parser.add_argument('diff_id', type=int)
|
|
||||||
parser.add_argument('--path', type=str, help='repository path', default=os.getcwd())
|
|
||||||
parser.add_argument('--token', type=str, default=None, help='Conduit API token')
|
|
||||||
parser.add_argument('--url', type=str, default=None, help='Phabricator URL')
|
|
||||||
parser.add_argument('--commit', dest='commit', type=str, default='auto',
|
|
||||||
help='Use this commit as a base. For "auto" tool tries to pick the base commit itself')
|
|
||||||
parser.add_argument('--push-branch', action='store_true', dest='push_branch',
|
|
||||||
help='choose if branch shall be pushed to origin')
|
|
||||||
parser.add_argument('--phid', type=str, default=None, help='Phabricator ID of the review this commit pertains to')
|
|
||||||
parser.add_argument('--log-level', type=str, default='INFO')
|
|
||||||
args = parser.parse_args()
|
|
||||||
logging.basicConfig(level=args.log_level, format='%(levelname)-7s %(message)s')
|
|
||||||
patcher = ApplyPatch(args.path, args.diff_id, args.token, args.url, args.commit, args.phid, args.push_branch)
|
|
||||||
sys.exit(patcher.run())
|
|
||||||
Loading…
Reference in a new issue