From 90e7224ba984f2be228f1506af23794f4b8f3816 Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Tue, 3 Nov 2020 15:51:59 +0100 Subject: [PATCH] improvements to patch process - improve messaging and add instructions how to fix the patch - fix application of patch stack. Now base commit and order of patches should be correct - set origin to llvm-project fork to avoid accidental operations on origin - use annotations to make issues visible - create commits with original author of change - add patches we tried to apply to artifacts - patching / commits works locally if --push-branch=false is set - misc renames / wordings --- scripts/apply_patch.sh | 4 +- scripts/buildkite_utils.py | 35 +++- scripts/patch_diff.py | 348 +++++++++++++++++++++++++++++++ scripts/phabtalk/apply_patch2.py | 323 ---------------------------- 4 files changed, 384 insertions(+), 326 deletions(-) create mode 100755 scripts/patch_diff.py delete mode 100755 scripts/phabtalk/apply_patch2.py diff --git a/scripts/apply_patch.sh b/scripts/apply_patch.sh index 2d845f5..d0899ba 100755 --- a/scripts/apply_patch.sh +++ b/scripts/apply_patch.sh @@ -18,7 +18,7 @@ set -uo pipefail -scripts/phabtalk/apply_patch2.py $ph_buildable_diff \ +scripts/patch_diff.py $ph_buildable_diff \ --path "${BUILDKITE_BUILD_PATH}"/llvm-project-fork \ --token $CONDUIT_TOKEN \ --url $PHABRICATOR_HOST \ @@ -30,7 +30,7 @@ scripts/phabtalk/apply_patch2.py $ph_buildable_diff \ EXIT_STATUS=$? if [ $EXIT_STATUS -ne 0 ]; then - scripts/add_phabricator_artifact.py --phid="$ph_target_phid" --url="$BUILDKITE_BUILD_URL" --name="Buildkite apply patch" + scripts/add_phabricator_artifact.py --phid="$ph_target_phid" --url="$BUILDKITE_BUILD_URL" --name="patch application failed" scripts/set_build_status.py echo failed fi diff --git a/scripts/buildkite_utils.py b/scripts/buildkite_utils.py index 9696c87..74f139f 100644 --- a/scripts/buildkite_utils.py +++ b/scripts/buildkite_utils.py @@ -2,11 +2,16 @@ import logging import os import re import subprocess +import urllib.parse +import shlex from typing import Optional import backoff import requests +context_style = {} +styles = ['default', 'info', 'success', 'warning', 'error'] + def upload_file(base_dir: str, file: str): """ @@ -25,6 +30,34 @@ def upload_file(base_dir: str, file: str): return None +def annotate(message: str, style: str = 'default', context: str = 'default', append: bool = True): + """ + Adds an annotation for that currently running build. + Note that last `style` applied to the same `context` takes precedence. + """ + if style not in styles: + style = 'default' + # Pick most severe style so far. + context_style.setdefault(context, 0) + context_style[context] = max(styles.index(style), context_style[context]) + style = styles[context_style[context]] + if append: + message += '\n\n' + r = subprocess.run(f"buildkite-agent annotate {shlex.quote(message)}" + f' --style={shlex.quote(style)}' + f" {'--append' if append else ''}" + f" --context={shlex.quote(context)}", shell=True, capture_output=True) + logging.debug(f'annotate call {r}') + if r.returncode != 0: + logging.warning(message) + + +def feedback_url(): + title = f"buildkite build {os.getenv('BUILDKITE_PIPELINE_SLUG')} {os.getenv('BUILDKITE_BUILD_NUMBER')}" + return f'https://github.com/google/llvm-premerge-checks/issues/new?assignees=&labels=bug' \ + f'&template=bug_report.md&title={urllib.parse.quote(title)}' + + class BuildkiteApi: def __init__(self, token: str, organization: str): self.token = token @@ -37,7 +70,7 @@ class BuildkiteApi: url = f'https://api.buildkite.com/v2/organizations/{self.organization}/pipelines/{pipeline}/builds/{build_number}' response = requests.get(url, headers={'Authorization': authorization}) if response.status_code != 200: - raise Exception(f'Builkite responded with non-OK status: {re.status_code}') + raise Exception(f'Builkite responded with non-OK status: {response.status_code}') return response.json() diff --git a/scripts/patch_diff.py b/scripts/patch_diff.py new file mode 100755 index 0000000..737fc3c --- /dev/null +++ b/scripts/patch_diff.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +# Copyright 2019 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import datetime +import logging +import os +import re +import subprocess +import sys +from typing import List, Optional, Tuple, Dict + +import backoff +from buildkite_utils import annotate, feedback_url, upload_file +import git +from phabricator import Phabricator + +"""URL of upstream LLVM repository.""" +LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project' +FORK_REMOTE_URL = 'ssh://git@github.com/llvm-premerge-tests/llvm-project' + +"""How far back the script searches in the git history to find Revisions that +have already landed. """ +APPLIED_SCAN_LIMIT = datetime.timedelta(days=90) + + +class ApplyPatch: + """Apply a diff from Phabricator on local working copy. + + This script is a rewrite of `arc patch` to accommodate for dependencies + that have already landed, but could not be identified by `arc patch`. + + For a given diff_id, this class will get the dependencies listed on Phabricator. + For each dependency D it will check the diff history: + - if D has already landed, skip it. + - If D has not landed, it will download the patch for D and try to apply it locally. + Once this class has applied all dependencies, it will apply the original diff. + + This script must be called from the root folder of a local checkout of + https://github.com/llvm/llvm-project or given a path to clone into. + """ + + def __init__(self, path: str, diff_id: int, token: str, url: str, git_hash: str, + phid: str, push_branch: bool = False): + self.push_branch = push_branch # type: bool + self.conduit_token = token # type: Optional[str] + self.host = url # type: Optional[str] + self.diff_id = diff_id # type: int + self.phid = phid # type: str + if not self.host.endswith('/api/'): + self.host += '/api/' + self.phab = self.create_phab() + self.base_revision = git_hash # type: str + self.branch_base_hexsha = '' + self.apply_diff_counter = 0 + self.build_dir = os.getcwd() + self.revision_id = '' + + if not os.path.isdir(path): + logging.info(f'{path} does not exist, cloning repository...') + self.repo = git.Repo.clone_from(FORK_REMOTE_URL, path) + else: + logging.info('repository exist, will reuse') + self.repo = git.Repo(path) # type: git.Repo + self.repo.remote('origin').set_url(FORK_REMOTE_URL) + os.chdir(path) + logging.info(f'working dir {os.getcwd()}') + + @property + def branch_name(self): + """Name used for the git branch.""" + return f'phab-diff-{self.diff_id}' + + def run(self): + """try to apply the patch from phabricator + """ + try: + diff = self.get_diff(self.diff_id) + revision = self.get_revision(diff.revisionID) + url = f"https://reviews.llvm.org/D{revision['id']}?id={diff['id']}" + annotate(f"Patching changes [{url}]({url})", style='info', context='patch_diff') + self.reset_repository() + self.revision_id = revision['id'] + dependencies = self.get_dependencies(revision) + dependencies.reverse() # Now revisions will be from oldest to newest. + missing, landed = self.classify_revisions(dependencies) + if len(dependencies) > 0: + logging.info('This diff depends on: {}'.format(revision_list_to_str(dependencies))) + logging.info(' Already landed: {}'.format(revision_list_to_str(landed))) + logging.info(' Will be applied: {}'.format(revision_list_to_str(missing))) + plan = [] + for r in missing: + d = self.get_diff(r['diffs'][0]) + plan.append((r, d)) + plan.append((revision, diff)) + logging.info('Planning to apply in order:') + for (r, d) in plan: + logging.info(f"https://reviews.llvm.org/D{r['id']}?id={d['id']}") + # Pick the newest known commit as a base for patches. + base_commit = None + for (r, d) in plan: + c = self.find_commit(d['sourceControlBaseRevision']) + if c is None: + logging.warning(f"D{r['id']}#{d['id']} commit {d['sourceControlBaseRevision']} does not exist") + continue + if base_commit is None: + logging.info(f"D{r['id']}#{d['id']} commit {c.hexsha} exists") + base_commit = c + elif c.committed_datetime > base_commit.committed_datetime: + logging.info(f"D{r['id']}#{d['id']} commit {c.hexsha} has a later commit date then" + f"{base_commit.hexsha}") + base_commit = c + if self.base_revision != 'auto': + logging.info(f'Base revision "{self.base_revision}" is set by command argument. Will use ' + f'instead of resolved "{base_commit}"') + base_commit = self.find_commit(self.base_revision) + if base_commit is None: + base_commit = self.repo.heads['master'].commit + annotate(f"Cannot find a base git revision. Will use current HEAD.", + style='warning', context='patch_diff') + self.create_branch(base_commit) + for (r, d) in plan: + if not self.apply_diff(d, r): + return 1 + if self.push_branch: + self.repo.git.push('--force', 'origin', self.branch_name) + annotate(f"Created branch [{self.branch_name}]" + f"(https://github.com/llvm-premerge-tests/llvm-project/tree/{self.branch_name}).\n\n" + f"To checkout locally, run in your copy of llvm-project directory:\n\n" + "```shell\n" + "git remote add premerge git@github.com:llvm-premerge-tests/llvm-project.git #first time\n" + f"git fetch premerge {self.branch_name}\n" + f"git checkout -b {self.branch_name} --track premerge/{self.branch_name}\n" + "```", + style='success', + context='patch_diff') + logging.info('Branch {} has been pushed'.format(self.branch_name)) + return 0 + except Exception as e: + annotate(f":bk-status-failed: Unexpected error. Consider [creating a bug]({feedback_url()}).", + style='error', context='patch_diff') + logging.error(f'exception: {e}') + return 1 + + @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) + def reset_repository(self): + """Update local git repo and origin. + + As origin is disjoint from upstream, it needs to be updated by this script. + """ + logging.info('Syncing local, origin and upstream...') + self.repo.git.clean('-ffxdq') + self.repo.git.reset('--hard') + self.repo.git.fetch('--all') + self.repo.git.checkout('master') + if 'upstream' not in self.repo.remotes: + self.repo.create_remote('upstream', url=LLVM_GITHUB_URL) + self.repo.remotes.upstream.fetch() + self.repo.git.pull('origin', 'master') + self.repo.git.pull('upstream', 'master') + if self.push_branch: + self.repo.git.push('origin', 'master') + + @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) + def find_commit(self, rev): + try: + return self.repo.commit(rev) + except ValueError as e: + return None + + @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) + def create_branch(self, base_commit: git.Commit): + if self.branch_name in self.repo.heads: + self.repo.delete_head('--force', self.branch_name) + logging.info(f'creating branch {self.branch_name} at {base_commit.hexsha}') + new_branch = self.repo.create_head(self.branch_name, base_commit.hexsha) + self.repo.head.reference = new_branch + self.repo.head.reset(index=True, working_tree=True) + self.branch_base_hexsha = self.repo.head.commit.hexsha + logging.info('Base branch revision is {}'.format(self.repo.head.commit.hexsha)) + annotate(f"Branch {self.branch_name} base revision is `{self.branch_base_hexsha}`.", + style='info', context='patch_diff') + + @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) + def commit(self, revision: Dict, diff: Dict): + """Commit the current state and annotates with the revision info.""" + self.repo.git.add('-A') + diff.setdefault('authorName', 'unknown') + diff.setdefault('authorEmail', 'unknown') + author = git.Actor(name=diff['authorName'], email=diff['authorEmail']) + message = (f"{revision['title']}\n\n" + f"Automated commit created by applying diff {self.diff_id}\n" + f"\n" + f"Phabricator-ID: {self.phid}\n" + f"Review-ID: {diff_to_str(revision['id'])}\n") + self.repo.index.commit(message=message, author=author) + + @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) + def create_phab(self): + phab = Phabricator(token=self.conduit_token, host=self.host) + phab.update_interfaces() + return phab + + @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) + def get_diff(self, diff_id: int): + """Get a diff from Phabricator based on it's diff id.""" + return self.phab.differential.getdiff(diff_id=diff_id) + + @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) + def get_revision(self, revision_id: int): + """Get a revision from Phabricator based on its revision id.""" + return self.phab.differential.query(ids=[revision_id])[0] + + @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) + def get_revisions(self, *, phids: List[str] = None): + """Get a list of revisions from Phabricator based on their PH-IDs.""" + if phids is None: + raise Exception('_get_revisions phids is None') + if not phids: + # Handle an empty query locally. Otherwise the connection + # will time out. + return [] + return self.phab.differential.query(phids=phids) + + def get_dependencies(self, revision: Dict) -> List[Dict]: + """Recursively resolves dependencies of the given revision. + They are listed in reverse chronological order - from most recent to least recent.""" + dependency_ids = revision['auxiliary']['phabricator:depends-on'] + revisions = self.get_revisions(phids=dependency_ids) + result = [] + for r in revisions: + result.append(r) + sub = self.get_dependencies(r) + result.extend(sub) + return result + + def apply_diff(self, diff: Dict, revision: Dict) -> bool: + """Download and apply a diff to the local working copy.""" + logging.info(f"Applying {diff['id']} for revision {revision['id']}...") + patch = self.get_raw_diff(str(diff['id'])) + self.apply_diff_counter += 1 + patch_file = f"{self.apply_diff_counter}_{diff['id']}.patch" + with open(os.path.join(self.build_dir, patch_file), 'wt') as f: + f.write(patch) + # For annotate to properly link this file it must exist before the upload. + upload_file(self.build_dir, patch_file) + logging.debug(f'raw patch:\n{patch}') + proc = subprocess.run('git apply -', input=patch, shell=True, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if proc.returncode != 0: + message = f":bk-status-failed: Failed to apply [{patch_file}](artifact://{patch_file}).\n\n" + if self.revision_id != revision['id']: + message += f"**Attention! D{revision['id']} is one of the dependencies of the target " \ + f"revision D{self.revision_id}.**\n\n" + message += (f"No testing is possible because we couldn't apply the patch.\n\n" + f"---\n\n" + '### Troubleshooting\n\n' + 'More information is available in the log of of *create branch* step. ' + f"All patches applied are available as *Artifacts*.\n\n" + f":bulb: The patch may not apply if it includes only the most recent of " + f"multiple local commits. Try to upload a patch with\n" + f"```shell\n" + f"arc diff `git merge-base HEAD origin` --update D{revision['id']}\n" + f"```\n\n" + f"to include all local changes.\n\n" + '---\n\n' + f"If this case could have been handled better, please [create a bug]({feedback_url()}).") + annotate(message, + style='error', + context='patch_diff') + return False + self.commit(revision, diff) + return True + + @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) + def get_raw_diff(self, diff_id: str) -> str: + return self.phab.differential.getrawdiff(diffID=diff_id).response + + def get_landed_revisions(self): + """Get list of landed revisions from current git branch.""" + diff_regex = re.compile(r'^Differential Revision: https://reviews\.llvm\.org/(.*)$', re.MULTILINE) + earliest_commit = None + rev = self.base_revision + age_limit = datetime.datetime.now() - APPLIED_SCAN_LIMIT + if rev == 'auto': # FIXME: use revison that created the branch + rev = 'master' + for commit in self.repo.iter_commits(rev): + if datetime.datetime.fromtimestamp(commit.committed_date) < age_limit: + break + earliest_commit = commit + result = diff_regex.search(commit.message) + if result is not None: + yield result.group(1) + if earliest_commit is not None: + logging.info(f'Earliest analyzed commit in history {earliest_commit.hexsha}, ' + f'{earliest_commit.committed_datetime}') + return + + def classify_revisions(self, revisions: List[Dict]) -> Tuple[List[Dict], List[Dict]]: + """Check which of the dependencies have already landed on the current branch.""" + landed_deps = [] + missing_deps = [] + for d in revisions: + if diff_to_str(d['id']) in self.get_landed_revisions(): + landed_deps.append(d) + else: + missing_deps.append(d) + return missing_deps, landed_deps + + +def diff_to_str(diff: int) -> str: + """Convert a diff id to a string with leading "D".""" + return 'D{}'.format(diff) + + +def revision_list_to_str(diffs: List[Dict]) -> str: + """Convert list of diff ids to a comma separated list, prefixed with "D".""" + return ', '.join([diff_to_str(d['id']) for d in diffs]) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Apply Phabricator patch to working directory.') + parser.add_argument('diff_id', type=int) + parser.add_argument('--path', type=str, help='repository path', default=os.getcwd()) + parser.add_argument('--token', type=str, default=None, help='Conduit API token') + parser.add_argument('--url', type=str, default='https://reviews.llvm.org', help='Phabricator URL') + parser.add_argument('--commit', dest='commit', type=str, default='auto', + help='Use this commit as a base. For "auto" tool tries to pick the base commit itself') + parser.add_argument('--push-branch', action='store_true', dest='push_branch', + help='choose if branch shall be pushed to origin') + parser.add_argument('--phid', type=str, default=None, help='Phabricator ID of the review this commit pertains to') + parser.add_argument('--log-level', type=str, default='INFO') + args = parser.parse_args() + logging.basicConfig(level=args.log_level, format='%(levelname)-7s %(message)s') + patcher = ApplyPatch(args.path, args.diff_id, args.token, args.url, args.commit, args.phid, args.push_branch) + sys.exit(patcher.run()) diff --git a/scripts/phabtalk/apply_patch2.py b/scripts/phabtalk/apply_patch2.py deleted file mode 100755 index d6dfe29..0000000 --- a/scripts/phabtalk/apply_patch2.py +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import datetime -import json -import logging -import os -import re -import socket -import subprocess -import sys -import time -from typing import List, Optional, Tuple - -import backoff -from git import Repo, BadName, GitCommandError -from phabricator import Phabricator - -# FIXME: maybe move to config file -"""URL of upstream LLVM repository.""" -LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project' -FORK_REMOTE_URL = 'ssh://git@github.com/llvm-premerge-tests/llvm-project' - -"""How far back the script searches in the git history to find Revisions that -have already landed. """ -APPLIED_SCAN_LIMIT = datetime.timedelta(days=90) - - -class ApplyPatch: - """Apply a diff from Phabricator on local working copy. - - This script is a rewrite of `arc patch` to accomodate for dependencies - that have already landed, but could not be identified by `arc patch`. - - For a given diff_id, this class will get the dependencies listed on Phabricator. - For each dependency D it will check the diff history: - - if D has already landed, skip it. - - If D has not landed, it will download the patch for D and try to apply it locally. - Once this class has applied all dependencies, it will apply the diff itself. - - This script must be called from the root folder of a local checkout of - https://github.com/llvm/llvm-project or given a path to clone into. - """ - - def __init__(self, path: str, diff_id: int, token: str, url: str, git_hash: str, - phid: str, push_branch: bool = False): - self.push_branch = push_branch # type: bool - self.conduit_token = token # type: Optional[str] - self.host = url # type: Optional[str] - self._load_arcrc() - self.diff_id = diff_id # type: int - self.phid = phid # type: str - if not self.host.endswith('/api/'): - self.host += '/api/' - self.phab = self._create_phab() - self.base_revision = git_hash # type: str - - if not os.path.isdir(path): - logging.info(f'{path} does not exist, cloning repository') - # TODO: progress of clonning - self.repo = Repo.clone_from(FORK_REMOTE_URL, path) - else: - logging.info('repository exist, will reuse') - self.repo = Repo(path) # type: Repo - # TODO: set origin url to fork - os.chdir(path) - logging.info(f'working dir {os.getcwd()}') - - @property - def branch_name(self): - """Name used for the git branch.""" - return 'phab-diff-{}'.format(self.diff_id) - - def _load_arcrc(self): - """Load arc configuration from file if not set.""" - if self.conduit_token is not None or self.host is not None: - return - logging.info('Loading configuration from ~/.arcrc file') - with open(os.path.expanduser('~/.arcrc'), 'r') as arcrc_file: - arcrc = json.load(arcrc_file) - # use the first host configured in the file - self.host = next(iter(arcrc['hosts'])) - self.conduit_token = arcrc['hosts'][self.host]['token'] - - def run(self): - """try to apply the patch from phabricator - - """ - - try: - self._refresh_master() - revision_id, dependencies, base_revision = self._get_dependencies(self.diff_id) - dependencies.reverse() # Arrange deps in chronological order. - if self.base_revision != 'auto': - logging.info('Using base revision provided by command line\n{} instead of resolved\n{}'.format( - self.base_revision, base_revision)) - base_revision = self.base_revision - self._create_branch(base_revision) - logging.info('git reset, git cleanup...') - self.repo.git.reset('--hard') - self.repo.git.clean('-ffxdq') - logging.info('Analyzing {}'.format(diff_to_str(revision_id))) - if len(dependencies) > 0: - logging.info('This diff depends on: {}'.format(diff_list_to_str(dependencies))) - missing, landed = self._get_missing_landed_dependencies(dependencies) - logging.info(' Already landed: {}'.format(diff_list_to_str(landed))) - logging.info(' Will be applied: {}'.format(diff_list_to_str(missing))) - if missing: - for revision in missing: - self._apply_revision(revision) - # FIXME: submit every Revision individually to get nicer history, use original user name - self.repo.config_writer().set_value("user", "name", "myusername").release() - self.repo.config_writer().set_value("user", "email", "myemail@example.com").release() - self.repo.git.commit('-a', '-m', 'dependencies') - logging.info('All depended diffs are applied') - logging.info('applying original diff') - self._apply_diff(self.diff_id, revision_id) - if self.push_branch: - self._commit_and_push(revision_id) - else: - self.repo.git.add('-u', '.') - return 0 - except Exception as e: - logging.error(f'exception: {e}') - return 1 - - @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) - def _refresh_master(self): - """Update local git repo and origin. - - As origin is disjoint from upstream, it needs to be updated by this script. - """ - logging.info('Syncing local, origin and upstream...') - self.repo.git.clean('-ffxdq') - self.repo.git.reset('--hard') - self.repo.git.fetch('--all') - self.repo.git.checkout('master') - if 'upstream' not in self.repo.remotes: - self.repo.create_remote('upstream', url=LLVM_GITHUB_URL) - self.repo.remotes.upstream.fetch() - self.repo.git.pull('origin', 'master') - self.repo.git.pull('upstream', 'master') - if self.push_branch: - self.repo.git.push('origin', 'master') - - @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) - def _create_branch(self, base_revision: Optional[str]): - self.repo.git.fetch('--all') - try: - if self.branch_name in self.repo.heads: - self.repo.delete_head('--force', self.branch_name) - except: - logging.warning('cannot delete branch {}'.format(self.branch_name)) - try: - commit = self.repo.commit(base_revision) - except Exception as e: - logging.info('Cannot resolve revision {}: {}, going to use "master" instead.'.format(base_revision, e)) - commit = self.repo.heads['master'].commit - logging.info(f'creating branch {self.branch_name} at {commit.hexsha}') - new_branch = self.repo.create_head(self.branch_name, commit.hexsha) - self.repo.head.reference = new_branch - self.repo.head.reset(index=True, working_tree=True) - logging.info('Base branch revision is {}'.format(self.repo.head.commit.hexsha)) - - @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) - def _commit_and_push(self, revision_id): - """Commit the patch and push it to origin.""" - if not self.push_branch: - return - - self.repo.git.add('-A') - message = """ -Applying diff {} - -Phabricator-ID: {} -Review-ID: {} -""".format(self.diff_id, self.phid, diff_to_str(revision_id)) - self.repo.index.commit(message=message) - self.repo.git.push('--force', 'origin', self.branch_name) - logging.info('Branch {} pushed to origin'.format(self.branch_name)) - pass - - @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) - def _create_phab(self): - phab = Phabricator(token=self.conduit_token, host=self.host) - phab.update_interfaces() - return phab - - @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) - def _get_diff(self, diff_id: int): - """Get a diff from Phabricator based on it's diff id.""" - return self.phab.differential.getdiff(diff_id=diff_id) - - @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) - def _get_revision(self, revision_id: int): - """Get a revision from Phabricator based on its revision id.""" - return self.phab.differential.query(ids=[revision_id])[0] - - @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) - def _get_revisions(self, *, phids: List[str] = None): - """Get a list of revisions from Phabricator based on their PH-IDs.""" - if phids is None: - raise Exception('_get_revisions phids is None') - if not phids: - # Handle an empty query locally. Otherwise the connection - # will time out. - return [] - return self.phab.differential.query(phids=phids) - - def _get_dependencies(self, diff_id) -> Tuple[int, List[int], str]: - """Get all dependencies for the diff. - They are listed in reverse chronological order - from most recent to least recent.""" - - logging.info('Getting dependencies of {}'.format(diff_id)) - diff = self._get_diff(diff_id) - logging.debug(f'diff object: {diff}') - revision_id = int(diff.revisionID) - revision = self._get_revision(revision_id) - logging.debug(f'revision object: {revision}') - base_revision = diff['sourceControlBaseRevision'] - if base_revision is None or len(base_revision) == 0: - base_revision = 'master' - dependency_ids = revision['auxiliary']['phabricator:depends-on'] - revisions = self._get_revisions(phids=dependency_ids) - result = [] - # Recursively resolve dependencies of those diffs. - for r in revisions: - _, sub, _ = self._get_dependencies(r['diffs'][0]) - result.append(r['id']) - result.extend(sub) - - return revision_id, result, base_revision - - @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) - def _apply_diff(self, diff_id: int, revision_id: int): - """Download and apply a diff to the local working copy.""" - logging.info('Applying diff {} for revision {}...'.format(diff_id, diff_to_str(revision_id))) - diff = self.phab.differential.getrawdiff(diffID=str(diff_id)).response - logging.debug(f'diff {diff_id}:\n{diff}') - proc = subprocess.run('git apply -', input=diff, shell=True, text=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - if proc.returncode != 0: - raise Exception('Applying patch failed:\n{}'.format(proc.stdout + proc.stderr)) - - def _apply_revision(self, revision_id: int): - """Download and apply the latest diff of a revision to the local working copy.""" - revision = self._get_revision(revision_id) - # take the diff_id with the highest number, this should be latest one - diff_id = max(revision['diffs']) - logging.info(f'picking diff from revision {revision_id}: {diff_id} from {revision["diffs"]}') - self._apply_diff(diff_id, revision_id) - - def _get_landed_revisions(self): - """Get list of landed revisions from current git branch.""" - diff_regex = re.compile(r'^Differential Revision: https://reviews\.llvm\.org/(.*)$', re.MULTILINE) - earliest_commit = None - rev = self.base_revision - age_limit = datetime.datetime.now() - APPLIED_SCAN_LIMIT - if rev == 'auto': # FIXME: use revison that created the branch - rev = 'master' - for commit in self.repo.iter_commits(rev): - if datetime.datetime.fromtimestamp(commit.committed_date) < age_limit: - break - earliest_commit = commit - result = diff_regex.search(commit.message) - if result is not None: - yield result.group(1) - if earliest_commit is not None: - logging.info(f'Earliest analyzed commit in history {earliest_commit.hexsha}, ' - f'{earliest_commit.committed_datetime}') - return - - def _get_missing_landed_dependencies(self, dependencies: List[int]) -> Tuple[List[int], List[int]]: - """Check which of the dependencies have already landed on the current branch.""" - landed_deps = [] - missing_deps = [] - for dependency in dependencies: - if diff_to_str(dependency) in self._get_landed_revisions(): - landed_deps.append(dependency) - else: - missing_deps.append(dependency) - return missing_deps, landed_deps - - -def diff_to_str(diff: int) -> str: - """Convert a diff id to a string with leading "D".""" - return 'D{}'.format(diff) - - -def diff_list_to_str(diffs: List[int]) -> str: - """Convert list of diff ids to a comma separated list, prefixed with "D".""" - return ', '.join([diff_to_str(d) for d in diffs]) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Apply Phabricator patch to working directory.') - parser.add_argument('diff_id', type=int) - parser.add_argument('--path', type=str, help='repository path', default=os.getcwd()) - parser.add_argument('--token', type=str, default=None, help='Conduit API token') - parser.add_argument('--url', type=str, default=None, help='Phabricator URL') - parser.add_argument('--commit', dest='commit', type=str, default='auto', - help='Use this commit as a base. For "auto" tool tries to pick the base commit itself') - parser.add_argument('--push-branch', action='store_true', dest='push_branch', - help='choose if branch shall be pushed to origin') - parser.add_argument('--phid', type=str, default=None, help='Phabricator ID of the review this commit pertains to') - parser.add_argument('--log-level', type=str, default='INFO') - args = parser.parse_args() - logging.basicConfig(level=args.log_level, format='%(levelname)-7s %(message)s') - patcher = ApplyPatch(args.path, args.diff_id, args.token, args.url, args.commit, args.phid, args.push_branch) - sys.exit(patcher.run())