diff --git a/containers/buildkite-premerge-debian/post-checkout b/containers/buildkite-premerge-debian/post-checkout index 2395b6f..9a61431 100755 --- a/containers/buildkite-premerge-debian/post-checkout +++ b/containers/buildkite-premerge-debian/post-checkout @@ -16,7 +16,8 @@ set -eu -# Run prune as git will start to complain about +# Run prune as git will start to complain "warning: There are too many unreachable loose objects; run 'git prune' to remove them.". +# That happens as we routinely drop old branches. git prune if [ -f ".git/gc.log" ]; then echo ".git/gc.log exist" diff --git a/docs/playbooks.md b/docs/playbooks.md index 769da44..d75d12d 100644 --- a/docs/playbooks.md +++ b/docs/playbooks.md @@ -192,7 +192,18 @@ Most commonly used are: - `ph_log_level` ("DEBUG", "INFO", "WARNING" (default) or "ERROR"): log level for build scripts. - `ph_linux_agents`, `ph_windows_agents`: custom JSON constraints on agents. For example, you might put one machine to a custom queue if it's errornous and send jobs to it with `ph_windows_agents={"queue": "custom"}`. - `ph_skip_linux`, `ph_skip_windows` (if set to any value): skip build on this OS. -- `ph_skip_generated`: don't run custom steps generated from within llvm-project. +- `ph_skip_generated`: don't run custom steps generated from within llvm-project. + +While trying a new patch for premerge scripts it's typical to start a new build by copying "ph_" +env variables from one of the recent builds and appending + +```shell +ph_dry_run_report=yes +ph_skip_windows=yes +ph_skip_generated=yes +ph_scripts_refspec="" +ph_log_level=DEBUG +``` ## Update HTTP auth credentials diff --git a/scripts/choose_projects.py b/scripts/choose_projects.py index 3987388..8001847 100755 --- a/scripts/choose_projects.py +++ b/scripts/choose_projects.py @@ -122,7 +122,8 @@ class ChooseProjects: @staticmethod def get_changed_files(patch_str: str = None) -> Set[str]: - """get list of changed files from the patch or from STDIN.""" + """get list of changed files from the patch or from STDIN. + e.g. ['compiler-rt/lib/tsan/CMakeLists.txt']""" if patch_str is None: patch_str = sys.stdin patch = PatchSet(patch_str) diff --git a/scripts/patch_diff.py b/scripts/patch_diff.py index a24a422..8fca27f 100755 --- a/scripts/patch_diff.py +++ b/scripts/patch_diff.py @@ -219,7 +219,7 @@ class ApplyPatch: @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) def get_diff(self, diff_id: int): - """Get a diff from Phabricator based on it's diff id.""" + """Get a diff from Phabricator based on its diff id.""" return self.phab.differential.getdiff(diff_id=diff_id) @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) diff --git a/scripts/phabtalk/phabtalk.py b/scripts/phabtalk/phabtalk.py index aadc5d6..9db671b 100755 --- a/scripts/phabtalk/phabtalk.py +++ b/scripts/phabtalk/phabtalk.py @@ -23,11 +23,13 @@ import argparse import backoff from phabricator import Phabricator +from benedict import benedict class PhabTalk: """Talk to Phabricator to upload build results. See https://secure.phabricator.com/conduit/method/harbormaster.sendmessage/ + You might want to use it as it provides retries on most of the calls. """ def __init__(self, token: Optional[str], host: Optional[str] = 'https://reviews.llvm.org/api/', @@ -47,6 +49,11 @@ class PhabTalk: result = self._phab.differential.querydiffs(ids=[diff]) return 'D' + result[diff]['revisionID'] + @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) + def get_diff(self, diff_id: int): + """Get a diff from Phabricator based on its diff id.""" + return self._phab.differential.getdiff(diff_id=diff_id) + def comment_on_diff(self, diff_id: str, text: str): """Add a comment to a differential based on the diff_id""" logging.info('Sending comment to diff {}:'.format(diff_id)) @@ -144,6 +151,27 @@ class PhabTalk: return self.create_artifact(phid, str(uuid.uuid4()), 'uri', {'uri': url, 'ui.external': True, 'name': name}) + @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) + def user_projects(self, user_phid: str) -> List[str]: + """Returns slugs of all projects user has a membership.""" + projects = benedict(self._phab.project.search(constraints={'members': [user_phid]})) + slugs = [] + for p in projects.get('data', []): + slug = benedict(p).get('fields.slug') + if slug: + slugs.append(p['fields']['slug']) + return slugs + + @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) + def get_revision(self, revision_id: int): + """Get a revision from Phabricator based on its revision id.""" + return self._phab.differential.query(ids=[revision_id])[0] + + @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) + def get_diff(self, diff_id: int): + """Get a diff from Phabricator based on its diff id.""" + return self._phab.differential.getdiff(diff_id=diff_id) + class Step: def __init__(self, name=''): diff --git a/scripts/pipeline_create_branch.py b/scripts/pipeline_create_branch.py index a28c471..de14dc7 100755 --- a/scripts/pipeline_create_branch.py +++ b/scripts/pipeline_create_branch.py @@ -14,19 +14,31 @@ # limitations under the License. import os +import sys + import yaml import logging from buildkite_utils import set_metadata, BuildkiteApi - +from phabtalk.phabtalk import PhabTalk if __name__ == '__main__': diff_id = os.getenv("ph_buildable_diff") + revision_id = os.getenv("ph_buildable_revision") log_level = os.getenv('ph_log_level', 'INFO') base_commit = os.getenv('ph_base_commit', 'auto') run_build = os.getenv('ph_skip_build') is None trigger = os.getenv('ph_trigger_pipeline') logging.basicConfig(level=log_level, format='%(levelname)-7s %(message)s') + phabtalk = PhabTalk(os.getenv('CONDUIT_TOKEN'), dry_run_updates=(os.getenv('ph_dry_run_report') is not None)) + rev = phabtalk.get_revision(int(revision_id)) + user_id = rev.get('authorPHID') + logging.debug(f'authorPHID {user_id}') + if user_id is None: + logging.error('cannot find author of the revision') + sys.exit(1) + projects = phabtalk.user_projects(user_id) + logging.info(f'user projects: {", ".join(projects)}') # Cancel any existing builds. # Do this before setting own 'ph_buildable_revision'. try: @@ -43,8 +55,17 @@ if __name__ == '__main__': if trigger is None: trigger = 'premerge-checks' - steps = [] - steps.append({ + env = { + 'ph_scripts_refspec': '${BUILDKITE_BRANCH}', + 'ph_user_project_slugs': ",".join(projects), + # TODO: those two are for "apply_patch.sh". Maybe just look for "ph_" env in patch_diff.py? + 'LOG_LEVEL': log_level, + 'BASE_COMMIT': base_commit, + } + for e in os.environ: + if e.startswith('ph_'): + env[e] = os.getenv(e) + steps = [{ 'label': 'create branch', 'key': 'create-branch', 'commands': [ @@ -53,11 +74,8 @@ if __name__ == '__main__': ], 'agents': {'queue': 'service'}, 'timeout_in_minutes': 20, - 'env': { - 'LOG_LEVEL': log_level, - 'BASE_COMMIT': base_commit, - } - }) + 'env': env + }] if run_build: trigger_build_step = { 'trigger': trigger, @@ -66,14 +84,8 @@ if __name__ == '__main__': 'depends_on': 'create-branch', 'build': { 'branch': f'phab-diff-{diff_id}', - 'env': {}, + 'env': env, }, } - for e in os.environ: - if e.startswith('ph_'): - trigger_build_step['build']['env'][e] = os.getenv(e) - # Set scripts source from the current build if it's not yet defined. - if 'ph_scripts_refspec' not in trigger_build_step['build']['env']: - trigger_build_step['build']['env']['ph_scripts_refspec'] = '${BUILDKITE_BRANCH}' steps.append(trigger_build_step) print(yaml.dump({'steps': steps})) diff --git a/scripts/pipeline_premerge.py b/scripts/pipeline_premerge.py index e042fd1..5d09409 100755 --- a/scripts/pipeline_premerge.py +++ b/scripts/pipeline_premerge.py @@ -19,7 +19,7 @@ import os from buildkite_utils import annotate, feedback_url, set_metadata from choose_projects import ChooseProjects import git -from steps import generic_linux, generic_windows, from_shell_output, checkout_scripts +from steps import generic_linux, generic_windows, from_shell_output, checkout_scripts, bazel import yaml steps_generators = [ @@ -73,6 +73,7 @@ if __name__ == '__main__': if os.getenv('ph_skip_generated') is None: for gen in steps_generators: steps.extend(from_shell_output(gen)) + steps.extend(bazel(modified_files)) if phid is None: logging.warning('ph_target_phid is not specified. Skipping "Report" step') diff --git a/scripts/steps.py b/scripts/steps.py index e22a088..4bd51e4 100644 --- a/scripts/steps.py +++ b/scripts/steps.py @@ -17,7 +17,7 @@ import io import json import logging import os -from typing import List +from typing import List, Set from exec_utils import watch_shell import yaml @@ -75,6 +75,40 @@ def generic_linux(projects: str, check_diff: bool) -> List: return [linux_buld_step] +def bazel(modified_files: Set[str]) -> List: + if os.getenv('ph_skip_bazel') is not None: + logging.info('bazel build is skipped as "ph_skip_bazel" is set') + return [] + updated_build = any(s.startswith('utils/bazel/') for s in modified_files) + if updated_build: + logging.info('files in utils/bazel/ modified, will trigger bazel build') + else: + user_projects = os.getenv('ph_user_project_slugs', '').split(',') + if 'bazel_build' not in user_projects: + logging.info('bazel build is skipped as "bazel_build" is not listed in user projects and no files in ' + 'utils/bazel/ are modified') + return [] + agents = {'queue': 'llvm-bazel'} + t = os.getenv('ph_bazel_agents') + if t is not None: + agents = json.loads(t) + return [{ + 'label': ':bazel: bazel', + 'key': 'bazel', + 'commands': [ + 'set -eu', + 'cd utils/bazel', + 'bazel query //... + @llvm-project//... | xargs bazel test --config=generic_clang --config=rbe --test_output=errors --test_tag_filters=-nobuildkite --build_tag_filters=-nobuildkite', + ], + 'agents': agents, + 'timeout_in_minutes': 120, + 'retry': {'automatic': [ + {'exit_status': -1, 'limit': 2}, # Agent lost + {'exit_status': 255, 'limit': 2}, # Forced agent shutdown + ]}, + }] + + def generic_windows(projects: str) -> List: if os.getenv('ph_skip_windows') is not None: return [] diff --git a/scripts/summary.py b/scripts/summary.py index a4509b5..8524cbf 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -75,7 +75,7 @@ if __name__ == '__main__': for i, job in enumerate(build.get('jobs', [])): job = benedict(job) job_web_url = job.get('web_url', os.getenv('BUILDKITE_BUILD_URL', '')) - logging.info(f'{job.get("id")} state {job.get("state")}') + logging.info(f'{job.get("id")} {job.get("name")} state {job.get("state")}') job_state = job.get('state') if job.get('type') == 'waiter': continue