From 99e43a8d9a9e20d03b7a98ef800254f8ed3fe3e8 Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Wed, 9 Dec 2020 17:23:01 +0100 Subject: [PATCH] rename master to main branch following LLVM --- README.md | 10 ++--- containers/agent-windows-vs2019/Dockerfile | 2 +- containers/base-debian/Dockerfile | 2 +- docs/development.md | 6 +-- docs/playbooks.md | 8 ++-- phabricator-proxy/main.py | 2 +- scripts/benchmark.py | 2 +- scripts/clang_tidy_report.py | 4 +- scripts/metrics/README.md | 12 +++--- scripts/metrics/analyze_jobs.ipynb | 4 +- scripts/metrics/buildkite_master_stats.py | 4 +- scripts/metrics/repo.py | 2 +- scripts/metrics/repo_hist.py | 6 +-- scripts/patch_diff.py | 24 ++++++----- scripts/phab2github/phab2github.py | 28 ++++++------- scripts/phabtalk/apply_patch.py | 6 +-- scripts/pipeline_main.py | 49 ++++++++++++++++++++++ scripts/pipeline_master.py | 2 +- scripts/pipeline_premerge.py | 2 +- scripts/steps.py | 4 +- 20 files changed, 116 insertions(+), 63 deletions(-) create mode 100755 scripts/pipeline_main.py diff --git a/README.md b/README.md index bca1e1e..dc30fc5 100644 --- a/README.md +++ b/README.md @@ -13,14 +13,14 @@ Phabricator](https://reviews.llvm.org) instance. *Phabricator* (https://reviews.llvm.org) is the code review tool in the LLVM project. -The workflow checks the patches before a user merges them to the master branch - +The workflow checks the patches before a user merges them to the main branch - thus the term *pre-merge testing**. When a user uploads a patch to the LLVM Phabricator, Phabricator triggers the checks and then displays the results. -The CI system checks the patches **before** a user merges them to the master +The CI system checks the patches **before** a user merges them to the main branch. This way bugs in a patch are contained during the code review stage and -do not pollute the master branch. The more bugs the CI system can catch during -the code review phase, the more stable and bug-free the master branch will +do not pollute the main branch. The more bugs the CI system can catch during +the code review phase, the more stable and bug-free the main branch will become. [citation needed]() This repository contains the configurations and script to run pre-merge checks @@ -58,7 +58,7 @@ please make sure that either: * You set a git hash as `sourceControlBaseRevision` in Phabricator which is * available on the Github repository, **or** you define the dependencies of your -* patch in Phabricator, **or** your patch can be applied to the master branch. +* patch in Phabricator, **or** your patch can be applied to the main branch. Only then can the build server apply the patch locally and run the builds and tests. diff --git a/containers/agent-windows-vs2019/Dockerfile b/containers/agent-windows-vs2019/Dockerfile index 69b7cd2..9129814 100644 --- a/containers/agent-windows-vs2019/Dockerfile +++ b/containers/agent-windows-vs2019/Dockerfile @@ -46,7 +46,7 @@ RUN choco install -y sccache RUN pip install psutil # install python dependencies for the scripts -RUN pip install -r https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/requirements.txt +RUN pip install -r https://raw.githubusercontent.com/google/llvm-premerge-checks/main/scripts/requirements.txt # configure Python encoding ENV PYTHONIOENCODING=UTF-8 diff --git a/containers/base-debian/Dockerfile b/containers/base-debian/Dockerfile index c9ff238..d89e374 100644 --- a/containers/base-debian/Dockerfile +++ b/containers/base-debian/Dockerfile @@ -35,7 +35,7 @@ ENV LC_ALL en_US.UTF-8 # Install python dependencies for the scripts. ADD will check contentents of a file for changes changed. # TODO: that should be done during the build as it will pull this repo anyway and will have latest version. -ADD "https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/requirements.txt" requirements.txt +ADD "https://raw.githubusercontent.com/google/llvm-premerge-checks/main/scripts/requirements.txt" requirements.txt RUN pip3 install -r requirements.txt RUN ln -s /usr/bin/clang-10 /usr/bin/clang;\ diff --git a/docs/development.md b/docs/development.md index 54dfdf7..b08f0ec 100644 --- a/docs/development.md +++ b/docs/development.md @@ -25,7 +25,7 @@ build request. Buildkite job sends build results directly to Phabricator. - every review creates a new branch in [fork of llvm-project](https://github.com/llvm-premerge-tests/llvm-project). -![deployment diagram](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/google/llvm-premerge-checks/master/docs/deployment.plantuml) +![deployment diagram](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/google/llvm-premerge-checks/main/docs/deployment.plantuml) # Phabricator integration @@ -57,7 +57,7 @@ export SCRIPT_DIR="${SRC}"/scripts rm -rf "${SRC}" git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "${SRC}" cd "${SRC}" -git fetch origin "${ph_scripts_refspec:-master}":x +git fetch origin "${ph_scripts_refspec:-main}":x git checkout x cd "$BUILDKITE_BUILD_CHECKOUT_PATH" ${SCRIPT_DIR}/buildkite/build_branch_pipeline.py | tee /dev/tty | buildkite-agent pipeline upload @@ -75,7 +75,7 @@ That in sends an HTTP POST request to [**phab-proxy**](../phabricator-proxy) that submits a new buildkite job **diff-checks**. All parameters from the original request are put in the build's environment with `ph_` prefix (to avoid shadowing any Buildkite environment variable). "ph_scripts_refspec" parameter -defines refspec of llvm-premerge-checks to use ("master" by default). +defines refspec of llvm-premerge-checks to use ("main" by default). **diff-checks** pipeline ([create_branch_pipeline.py](../scripts/create_branch_pipeline.py)) diff --git a/docs/playbooks.md b/docs/playbooks.md index b5bbb40..78ad398 100644 --- a/docs/playbooks.md +++ b/docs/playbooks.md @@ -32,7 +32,7 @@ Set `CONDUIT_TOKEN` with your personal one from `https://reviews.llvm.org/settin ## Testing changes before merging -It's recommended to test even smallest changes before committing them to the `master` branch. +It's recommended to test even smallest changes before committing them to the `main` branch. 1. Create a pull request here. 1. Manually create a buildkite build in the pipeline you are updating and specify @@ -144,7 +144,7 @@ To spawn a new windows agent: 1. Login to the new machine via RDP (you will need a RDP client, e.g. Chrome app). 1. In the RDP session: run these commands in the CMD window under Administrator to bootstrap the Windows machine: ```powershell - Invoke-WebRequest -uri 'https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/windows_agent_bootstrap.ps1' -OutFile c:\windows_agent_bootstrap.ps1 + Invoke-WebRequest -uri 'https://raw.githubusercontent.com/google/llvm-premerge-checks/main/scripts/windows_agent_bootstrap.ps1' -OutFile c:\windows_agent_bootstrap.ps1 c:/windows_agent_bootstrap.ps1 -ssd ``` Ignore the pop-up to format the new disk and wait for the machine to reboot. @@ -173,12 +173,12 @@ schtasks.exe /create /tn "Start Buildkite agent" /ru SYSTEM /SC ONSTART /DELAY 0 ## Custom environment variables Buildkite pipelines have a number of custom environment variables one can set to change their behavior. That is useful to debug issues -or test changes. They are mostly used by pipleine generators, e.g. [build_master_pipeline](../scripts/build_master_pipeline.py), +or test changes. They are mostly used by pipleine generators, e.g. [pipeline_main](../scripts/pipeline_main.py), please refer to the source code for the details. These variables have `ph_` prefix and can be set with URL parameters in Harbormaster build. Most commonly used are: -- `ph_scripts_refspec` ("master" by default): refspec branch of llvm-premerge-checks to use. This variable is also used in pipeline "bootstrap" in Buildkite interface. +- `ph_scripts_refspec` ("main" by default): refspec branch of llvm-premerge-checks to use. This variable is also used in pipeline "bootstrap" in Buildkite interface. - `ph_dry_run_report`: do not report any results back to Phabricator. - `ph_no_cache`: (if set to any value) clear compilation cache before the build. - `ph_projects`: which projects to use, "detect" will look on diff to infer the projects, "default" selects all projects. diff --git a/phabricator-proxy/main.py b/phabricator-proxy/main.py index 4662e4c..c70a3ff 100644 --- a/phabricator-proxy/main.py +++ b/phabricator-proxy/main.py @@ -27,7 +27,7 @@ def build(): for k, v in params.items(): if len(v) == 1: build_env['ph_' + k] = v[0] - refspec = 'master' + refspec = 'main' if 'ph_scripts_refspec' in build_env: refspec = build_env['ph_scripts_refspec'] build_request = { diff --git a/scripts/benchmark.py b/scripts/benchmark.py index 1ffa417..dc1b5d8 100755 --- a/scripts/benchmark.py +++ b/scripts/benchmark.py @@ -140,7 +140,7 @@ if __name__ == '__main__': pmt_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) parser = argparse.ArgumentParser( description='Benchmark for LLVM pre-merge tests.') - parser.add_argument('--commit', type=str, default='master', help="LLVM commit to run this benchmark on.") + parser.add_argument('--commit', type=str, default='main', help="LLVM commit to run this benchmark on.") parser.add_argument('--result-file', type=str, default='pmt-benchmark.csv', help="path to CSV file where to store the benchmark results") parser.add_argument('--workdir', type=str, default=os.path.join(os.getcwd(), 'benchmark'), diff --git a/scripts/clang_tidy_report.py b/scripts/clang_tidy_report.py index e64e081..9a98ed5 100755 --- a/scripts/clang_tidy_report.py +++ b/scripts/clang_tidy_report.py @@ -72,7 +72,7 @@ def run(base_commit, ignore_config, step: Optional[Step], report: Optional[Repor severity = match.group(4) text = match.group(5) text += '\n[[{} | not useful]] '.format( - 'https://github.com/google/llvm-premerge-checks/blob/master/docs/clang_tidy.md#warning-is-not-useful') + 'https://github.com/google/llvm-premerge-checks/blob/main/docs/clang_tidy.md#warning-is-not-useful') if severity in ['warning', 'error']: if severity == 'warning': warn_count += 1 @@ -100,7 +100,7 @@ def run(base_commit, ignore_config, step: Optional[Step], report: Optional[Repor report.add_artifact(os.getcwd(), p, 'clang-tidy') if errors_count + warn_count != 0: step.success = False - url = "https://github.com/google/llvm-premerge-checks/blob/master/docs/clang_tidy.md#review-comments." + url = "https://github.com/google/llvm-premerge-checks/blob/main/docs/clang_tidy.md#review-comments." annotate(f'clang-tidy found {errors_count} errors and {warn_count} warnings. {inline_comments} of them were ' f'added as review comments [why?]({url})', style='error') logging.debug(f'report: {report}') diff --git a/scripts/metrics/README.md b/scripts/metrics/README.md index b9e930f..917776e 100644 --- a/scripts/metrics/README.md +++ b/scripts/metrics/README.md @@ -5,22 +5,22 @@ a set of metrics. This doc will summarize the metrics and tools. All of the data shall be collected as time series, so that we can see changes over time. * Impact - The metrics we ultimately want to improve - * Percentage of [build-bot build](http://lab.llvm.org:8011/) on master + * Percentage of [build-bot build](http://lab.llvm.org:8011/) on main failing. (Buildbot_percentage_failing) - * Time to fix a broken master build: Time between start of failing builds + * Time to fix a broken main build: Time between start of failing builds until the build is fixed. (BuildBot_time_to_fix) * Percentage of Revisions on Phabricator where a broken build was fixed afterwards. This would indicate that a bug was found and fixed during the code review phase. (Premerge_fixes) - * Number of reverts on master. This indicates that something was broken on - master that slipped through the pre-merge tests or was submitted without + * Number of reverts on main. This indicates that something was broken on + main that slipped through the pre-merge tests or was submitted without any review. (Upstream_reverts) * Users and behavior - Interesting to see and useful to adapt our approach. - * Percentage of commits to master that went through Phabricator. + * Percentage of commits to main that went through Phabricator. * Number of participants in pre-merge tests. * Percentage of Revisions with pre-merge tests executed - * Number of 30-day active committers on master and Phabricator. + * Number of 30-day active committers on main and Phabricator. * Builds - See how the infrastructure is doing. * Time between upload of diff until build results available. diff --git a/scripts/metrics/analyze_jobs.ipynb b/scripts/metrics/analyze_jobs.ipynb index 4a1dd54..470afe4 100644 --- a/scripts/metrics/analyze_jobs.ipynb +++ b/scripts/metrics/analyze_jobs.ipynb @@ -201,8 +201,8 @@ "metadata": {}, "outputs": [], "source": [ - "# ds = jobs[jobs['pipeline'] == 'llvm-master-build'][jobs['step_key'] == 'windows'][jobs['state']=='passed'][~jobs['agent_name'].str.startswith('buildkite-')][jobs['started_at'] > np.datetime64('2020-01-22')]\n", - "ds = jobs[jobs['pipeline'] == 'llvm-master-build'][jobs['step_key'] == 'windows'][jobs['state']=='passed'][~jobs['agent_name'].str.startswith('buildkite-')][jobs['started_at'] > np.datetime64('2020-01-22')]\n", + "# ds = jobs[jobs['pipeline'] == 'llvm-main-build'][jobs['step_key'] == 'windows'][jobs['state']=='passed'][~jobs['agent_name'].str.startswith('buildkite-')][jobs['started_at'] > np.datetime64('2020-01-22')]\n", + "ds = jobs[jobs['pipeline'] == 'llvm-main-build'][jobs['step_key'] == 'windows'][jobs['state']=='passed'][~jobs['agent_name'].str.startswith('buildkite-')][jobs['started_at'] > np.datetime64('2020-01-22')]\n", "ds = ds.drop_duplicates()\n", "# remove one slowest run (repo checkout)\n", "# t = ds.loc[ds.groupby([\"agent_name\"])[\"run_duration\"].idxmax()]\n", diff --git a/scripts/metrics/buildkite_master_stats.py b/scripts/metrics/buildkite_master_stats.py index f4af844..401c58f 100755 --- a/scripts/metrics/buildkite_master_stats.py +++ b/scripts/metrics/buildkite_master_stats.py @@ -14,7 +14,7 @@ # limitations under the License. # ----------------------------------------------------------------------------- -# This script will collect all breakages of the master branch builds from +# This script will collect all breakages of the main branch builds from # buildkite and format the results nicely. # Arguments: # llvm-path : folder where the LLVM checkout is kept @@ -107,7 +107,7 @@ if __name__ == '__main__': bk = BuildKiteMasterStats(args.llvm_path, args.token) if not os.path.exists(CACHE_FILE): - results = bk.get_stats('llvm-project','llvm-master-build') + results = bk.get_stats('llvm-project','llvm-main-build') bk.save_results(CACHE_FILE, results) builds = bk.get_builds(CACHE_FILE) diff --git a/scripts/metrics/repo.py b/scripts/metrics/repo.py index c8280fa..51c2385 100755 --- a/scripts/metrics/repo.py +++ b/scripts/metrics/repo.py @@ -70,7 +70,7 @@ def get_reverts_per_day(repo_path: str, max_age: datetime.datetime) -> RepoStats repo.git.fetch() diff_regex = re.compile(r'^Differential Revision: https:\/\/reviews\.llvm\.org\/(.*)$', re.MULTILINE) - for commit in repo.iter_commits('master'): + for commit in repo.iter_commits('main'): if commit.committed_datetime < max_age: break stats.commits += 1 diff --git a/scripts/metrics/repo_hist.py b/scripts/metrics/repo_hist.py index ed70827..4f8a6ba 100644 --- a/scripts/metrics/repo_hist.py +++ b/scripts/metrics/repo_hist.py @@ -121,7 +121,7 @@ class RepoStats: self.commit_by_author_domain = dict() # type: Dict[str, List[MyCommit]] def parse_repo(self, maxage: datetime.datetime): - for commit in self.repo.iter_commits('master'): + for commit in self.repo.iter_commits('main'): if commit.committed_datetime < maxage: break mycommit = MyCommit(commit) @@ -262,7 +262,7 @@ class RepoStats: True: {}, False: {}, } # type: Dict[bool, Dict[str, int]] - for commit in self.repo.iter_commits('master'): + for commit in self.repo.iter_commits('main'): if commit.committed_datetime < maxage: break mycommit = MyCommit(commit) @@ -295,7 +295,7 @@ class RepoStats: True: {b: 0 for b in buckets}, False: {b: 0 for b in buckets}, } # type: Dict[bool, Dict[int, int]] - for commit in self.repo.iter_commits('master'): + for commit in self.repo.iter_commits('main'): if commit.committed_datetime < maxage: break mycommit = self.commit_by_hash[commit.hexsha] diff --git a/scripts/patch_diff.py b/scripts/patch_diff.py index 4df3952..a24a422 100755 --- a/scripts/patch_diff.py +++ b/scripts/patch_diff.py @@ -127,7 +127,7 @@ class ApplyPatch: f'instead of resolved "{base_commit}"') base_commit = self.find_commit(self.base_revision) if base_commit is None: - base_commit = self.repo.heads['master'].commit + base_commit = self.repo.heads['main'].commit annotate(f"Cannot find a base git revision. Will use current HEAD.", style='warning', context='patch_diff') self.create_branch(base_commit) @@ -161,23 +161,27 @@ class ApplyPatch: As origin is disjoint from upstream, it needs to be updated by this script. """ logging.info('Syncing local, origin and upstream...') - self.repo.git.clean('-ffxdq') - self.repo.git.reset('--hard') - self.repo.git.fetch('--all') - self.repo.git.checkout('master') if 'upstream' not in self.repo.remotes: self.repo.create_remote('upstream', url=LLVM_GITHUB_URL) self.repo.remotes.upstream.fetch() - self.repo.git.pull('origin', 'master') - self.repo.git.pull('upstream', 'master') + self.repo.git.clean('-ffxdq') + self.repo.git.reset('--hard') + self.repo.git.fetch('--all') + if self.find_commit('main') is None: + origin = self.repo.remotes.origin + self.repo.create_head('main', origin.refs.main) + self.repo.heads.main.set_tracking_branch(origin.refs.main) + self.repo.heads.main.checkout() + self.repo.git.pull('origin', 'main') + self.repo.git.pull('upstream', 'main') if self.push_branch: - self.repo.git.push('origin', 'master') + self.repo.git.push('origin', 'main') @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) def find_commit(self, rev): try: return self.repo.commit(rev) - except ValueError as e: + except: return None @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) @@ -297,7 +301,7 @@ class ApplyPatch: rev = self.base_revision age_limit = datetime.datetime.now() - APPLIED_SCAN_LIMIT if rev == 'auto': # FIXME: use revison that created the branch - rev = 'master' + rev = 'main' for commit in self.repo.iter_commits(rev): if datetime.datetime.fromtimestamp(commit.committed_date) < age_limit: break diff --git a/scripts/phab2github/phab2github.py b/scripts/phab2github/phab2github.py index 2ba58f0..b6d0bfc 100644 --- a/scripts/phab2github/phab2github.py +++ b/scripts/phab2github/phab2github.py @@ -50,7 +50,7 @@ class Phab2Github: def sync(self): """Sync Phabricator to Github.""" _LOGGER.info('Starting sync...') - self._refresh_master() + self._refresh_main() self._delete_phab_branches() revisions = self.phab_wrapper.get_revisions() pull_requests = {p.title: p for p in self.github_repo.get_pulls(state='open')} @@ -70,11 +70,11 @@ class Phab2Github: pr = self.github_repo.create_pull(title=revision.pr_title, body=revision.pr_summary, head=revision.branch_name, - base='master') + base='main') _LOGGER.info(pr.html_url) _LOGGER.info('Sync completed.') - def _refresh_master(self): + def _refresh_main(self): """Clone/update local git repo.""" if not os.path.exists(self.workdir): os.mkdir(self.workdir) @@ -83,16 +83,16 @@ class Phab2Github: _LOGGER.info('pulling origin and upstream...') self.repo = git.Repo(self.llvm_dir) self.repo.git.fetch('--all') - self.repo.git.checkout('master') - self.repo.git.pull('upstream', 'master') - self.repo.git.push('origin', 'master') + self.repo.git.checkout('main') + self.repo.git.pull('upstream', 'main') + self.repo.git.push('origin', 'main') else: _LOGGER.info('cloning repository...') git.Repo.clone_from(MY_GITHUB_URL, self.llvm_dir) self.repo = git.Repo(self.llvm_dir) self.repo.create_remote('upstream', url=LLVM_GITHUB_URL) self.repo.remotes.upstream.fetch() - _LOGGER.info('refresh of master branch completed') + _LOGGER.info('refresh of main branch completed') def create_branches_for_revision(self, revision: Revision): """Create branches for a Revision and it's Diffs. @@ -110,10 +110,10 @@ class Phab2Github: try: self.apply_patch(diff, patch) except ApplyPatchException as e: - # TODO: retry on master if this fails + # TODO: retry on main if this fails _LOGGER.error('Could not apply patch for Diff {}. Deleting branch'.format(diff.id)) _LOGGER.exception(e) - self.repo.heads['master'].checkout() + self.repo.heads['main'].checkout() self.repo.delete_head(diff.branch_name) diffs = [d for d in revision.sorted_diffs if self._has_branch(d)] @@ -143,15 +143,15 @@ class Phab2Github: """Create a branch for diff.""" base_hash = diff.base_hash if base_hash is None: - base_hash = 'upstream/master' + base_hash = 'upstream/main' _LOGGER.info('creating branch {} based on {}...'.format(diff.branch_name, base_hash)) try: new_branch = self.repo.create_head(diff.branch_name, base_hash) except ValueError: - # commit hash not found, try again with master + # commit hash not found, try again with main _LOGGER.warning('commit hash {} not found in upstream repository. ' - 'Trying master instead...'.format(diff.branch_name, base_hash)) - base_hash = 'upstream/master' + 'Trying main instead...'.format(diff.branch_name, base_hash)) + base_hash = 'upstream/main' new_branch = self.repo.create_head(diff.branch_name, base_hash) self.repo.head.reference = new_branch self.repo.head.reset(index=True, working_tree=True) @@ -184,7 +184,7 @@ class Phab2Github: def _delete_phab_branches(self): """Delete all branches sarting with 'phab-'.""" _LOGGER.info('Deleting local Phabricator-relates branches...') - self.repo.git.checkout('master') + self.repo.git.checkout('main') for branch in [b for b in self.repo.heads if b.name.startswith('phab-')]: _LOGGER.info('Deleding branch {}'.format(branch)) self.repo.git.branch('-D', branch.name) diff --git a/scripts/phabtalk/apply_patch.py b/scripts/phabtalk/apply_patch.py index f4336c3..c590c0e 100755 --- a/scripts/phabtalk/apply_patch.py +++ b/scripts/phabtalk/apply_patch.py @@ -88,13 +88,13 @@ class ApplyPatch: subprocess.check_call('git reset --hard {}'.format(self.git_hash), stdout=sys.stdout, stderr=sys.stderr, shell=True) except subprocess.CalledProcessError: - print('WARNING: checkout of hash failed, using master branch instead.') + print('WARNING: checkout of hash failed, using main branch instead.') self.msg += [ 'Could not check out parent git hash "{}". It was not found in ' 'the repository. Did you configure the "Parent Revision" in ' 'Phabricator properly? Trying to apply the patch to the ' - 'master branch instead...'.format(self.git_hash)] - subprocess.check_call('git checkout master', stdout=sys.stdout, + 'main branch instead...'.format(self.git_hash)] + subprocess.check_call('git checkout main', stdout=sys.stdout, stderr=sys.stderr, shell=True) subprocess.check_call('git show -s', stdout=sys.stdout, stderr=sys.stderr, shell=True) diff --git a/scripts/pipeline_main.py b/scripts/pipeline_main.py new file mode 100755 index 0000000..6e98118 --- /dev/null +++ b/scripts/pipeline_main.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# Copyright 2020 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from steps import generic_linux, generic_windows, from_shell_output +import yaml + +steps_generators = [ + # TODO: enable when we have enough mac agents + # '${BUILDKITE_BUILD_CHECKOUT_PATH}/libcxx/utils/ci/buildkite-pipeline-snapshot.sh', +] + +if __name__ == '__main__': + scripts_refspec = os.getenv("ph_scripts_refspec", "main") + no_cache = os.getenv('ph_no_cache') is not None + projects = os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly') + log_level = os.getenv('ph_log_level', 'WARNING') + notify_emails = list(filter(None, os.getenv('ph_notify_emails', '').split(','))) + steps = [] + steps.extend(generic_linux( + os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly'), + False)) + # FIXME: openmp is removed as it constantly fails. + + # TODO: Make this project list be evaluated through "choose_projects"(? as now we define "all" and exclusions in + # two placess). + steps.extend(generic_windows( + os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly'))) + + for gen in steps_generators: + steps.extend(from_shell_output(gen)) + + notify = [] + for e in notify_emails: + notify.append({'email': e}) + print(yaml.dump({'steps': steps, 'notify': notify})) diff --git a/scripts/pipeline_master.py b/scripts/pipeline_master.py index 5ce5349..6e98118 100755 --- a/scripts/pipeline_master.py +++ b/scripts/pipeline_master.py @@ -24,7 +24,7 @@ steps_generators = [ ] if __name__ == '__main__': - scripts_refspec = os.getenv("ph_scripts_refspec", "master") + scripts_refspec = os.getenv("ph_scripts_refspec", "main") no_cache = os.getenv('ph_no_cache') is not None projects = os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly') log_level = os.getenv('ph_log_level', 'WARNING') diff --git a/scripts/pipeline_premerge.py b/scripts/pipeline_premerge.py index ee37a69..cace751 100755 --- a/scripts/pipeline_premerge.py +++ b/scripts/pipeline_premerge.py @@ -27,7 +27,7 @@ steps_generators = [ ] if __name__ == '__main__': - scripts_refspec = os.getenv("ph_scripts_refspec", "master") + scripts_refspec = os.getenv("ph_scripts_refspec", "main") diff_id = os.getenv("ph_buildable_diff", "") no_cache = os.getenv('ph_no_cache') is not None projects = os.getenv('ph_projects', 'detect') diff --git a/scripts/steps.py b/scripts/steps.py index bfb06f1..108bceb 100644 --- a/scripts/steps.py +++ b/scripts/steps.py @@ -26,7 +26,7 @@ import yaml def generic_linux(projects: str, check_diff: bool) -> List: if os.getenv('ph_skip_linux') is not None: return [] - scripts_refspec = os.getenv("ph_scripts_refspec", "master") + scripts_refspec = os.getenv("ph_scripts_refspec", "main") no_cache = os.getenv('ph_no_cache') is not None log_level = os.getenv('ph_log_level', 'WARNING') linux_agents = {'queue': 'linux'} @@ -79,7 +79,7 @@ def generic_linux(projects: str, check_diff: bool) -> List: def generic_windows(projects: str) -> List: if os.getenv('ph_skip_windows') is not None: return [] - scripts_refspec = os.getenv("ph_scripts_refspec", "master") + scripts_refspec = os.getenv("ph_scripts_refspec", "main") no_cache = os.getenv('ph_no_cache') is not None log_level = os.getenv('ph_log_level', 'WARNING') clear_sccache = 'powershell -command "sccache --stop-server; echo \\$env:SCCACHE_DIR; ' \