script to sync fork llvm-premerge-tests/llvm-project

- uniformly append env variables

- resolve current HEAD and pass it as BUILDKITE_COMMIT to script
generators. Add it as ph_commit_sha.

scripts/pipeline_main.py

@@ -13,9 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Script runs in checked out llvm-project directory.
+
 import os
 
-from steps import generic_linux, generic_windows, from_shell_output
+from steps import generic_linux, generic_windows, from_shell_output, extend_steps_env
+from sync_fork import sync_fork
+import git
 import yaml
 
 steps_generators = [
@@ -27,7 +31,18 @@ if __name__ == '__main__':
     no_cache = os.getenv('ph_no_cache') is not None
     log_level = os.getenv('ph_log_level', 'WARNING')
     notify_emails = list(filter(None, os.getenv('ph_notify_emails', '').split(',')))
+    # Syncing LLVM fork so any pipelines started from upstream llvm-project#
+    # but then triggered a build on fork will observe the commit.
+    sync_fork(os.path.join(os.getenv('BUILDKITE_BUILD_PATH'), 'llvm-project-fork'), [os.getenv('BUILDKITE_BRANCH'), 'main'])
     steps = []
+
+    env = {}
+    for e in os.environ:
+        if e.startswith('ph_'):
+            env[e] = os.getenv(e)
+    repo = git.Repo('.')
+    env['ph_commit_sha'] = repo.head.commit.hexsha
+
     steps.extend(generic_linux(
         os.getenv('ph_projects', 'llvm;clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly;flang'),
         False))
@@ -39,10 +54,14 @@ if __name__ == '__main__':
         os.getenv('ph_projects', 'llvm;clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly;flang')))
 
     if os.getenv('ph_skip_generated') is None:
+        e = os.environ.copy()
+        # BUILDKITE_COMMIT might be an alias, e.g. "HEAD". Resolve it to make the build hermetic.
+        e["BUILDKITE_COMMIT"] = repo.head.commit.hexsha
         for gen in steps_generators:
-            steps.extend(from_shell_output(gen))
+            steps.extend(from_shell_output(gen, env=e))
 
     notify = []
     for e in notify_emails:
         notify.append({'email': e})
+    extend_steps_env(steps, env)
     print(yaml.dump({'steps': steps, 'notify': notify}))

scripts/pipeline_premerge.py

@@ -13,13 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Script runs in checked out llvm-project directory.
+
 import logging
 import os
 
 from buildkite_utils import annotate, feedback_url, set_metadata
 from choose_projects import ChooseProjects
 import git
-from steps import generic_linux, generic_windows, from_shell_output, checkout_scripts, bazel
+from steps import generic_linux, generic_windows, from_shell_output, checkout_scripts, bazel, extend_steps_env
 import yaml
 
 steps_generators = [
@@ -42,8 +44,14 @@ if __name__ == '__main__':
     set_metadata('ph_buildable_diff', os.getenv("ph_buildable_diff"))
     set_metadata('ph_buildable_revision', os.getenv('ph_buildable_revision'))
     set_metadata('ph_build_id', os.getenv("ph_build_id"))
-    # List all affected projects.
+
+    env = {}
+    for e in os.environ:
+        if e.startswith('ph_'):
+            env[e] = os.getenv(e)
     repo = git.Repo('.')
+    env['ph_commit_sha'] = repo.head.commit.hexsha
+    # List all affected projects.
     patch = repo.git.diff("HEAD~1")
     cp = ChooseProjects('.')
     modified_files = cp.get_changed_files(patch)
@@ -71,8 +79,11 @@ if __name__ == '__main__':
         steps.extend(generic_windows(';'.join(sorted(windows_projects))))
     # Add custom checks.
     if os.getenv('ph_skip_generated') is None:
+        e = os.environ.copy()
+        # BUILDKITE_COMMIT might be an alias, e.g. "HEAD". Resolve it to make the build hermetic.
+        e["BUILDKITE_COMMIT"] = repo.head.commit.hexsha
         for gen in steps_generators:
-            steps.extend(from_shell_output(gen))
+            steps.extend(from_shell_output(gen, env=e))
     steps.extend(bazel(modified_files))
 
     if phid is None:
@@ -95,4 +106,5 @@ if __name__ == '__main__':
         }
         steps.append(report_step)
 
+    extend_steps_env(steps, env)
     print(yaml.dump({'steps': steps}))

scripts/steps.py

@@ -17,7 +17,7 @@ import io
 import json
 import logging
 import os
-from typing import List, Set
+from typing import List, Set, Dict
 
 from exec_utils import watch_shell
 import yaml
@@ -153,7 +153,7 @@ def generic_windows(projects: str) -> List:
     return [windows_buld_step]
 
 
-def from_shell_output(command) -> []:
+def from_shell_output(command, **kwargs) -> []:
     """
     Executes shell command and parses stdout as multidoc yaml file, see
     https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
@@ -165,7 +165,7 @@ def from_shell_output(command) -> []:
     logging.debug(f'invoking "{path}"')
     out = io.BytesIO()
     err = io.BytesIO()
-    rc = watch_shell(out.write, err.write, path)
+    rc = watch_shell(out.write, err.write, path, **kwargs)
     logging.debug(f'exit code: {rc}, stdout: "{out.getvalue().decode()}", stderr: "{err.getvalue().decode()}"')
     steps = []
     if rc != 0:
@@ -210,3 +210,21 @@ def checkout_scripts(target_os: str, scripts_refspec: str) -> []:
         'pip install -q -r $${SRC}/scripts/requirements.txt',
         'cd "$$BUILDKITE_BUILD_CHECKOUT_PATH"',
     ]
+
+
+def extend_dict(target: Dict, extra: Dict) -> Dict:
+    if not target:
+        return extra
+    for k in extra:
+        if k in target:
+            continue
+        target[k] = extra[k]
+    return target
+
+
+def extend_steps_env(steps: List[Dict], env: Dict):
+    for s in steps:
+        if 'commands' in s:
+            s['env'] = extend_dict(s.get('env'), env)
+        if 'build' in s:
+            s['build']['env'] = extend_dict(s['build'].get('env'), env)

scripts/sync_fork.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+# Copyright 2021 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import logging
+import os
+from typing import List
+import backoff
+import git
+
+"""URL of upstream LLVM repository."""
+LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project'
+FORK_REMOTE_URL = 'ssh://git@github.com/llvm-premerge-tests/llvm-project'
+
+
+@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
+def sync_fork(path: str, branches: List[str]):
+    if not os.path.isdir(path):
+        logging.info(f'{path} does not exist, cloning repository...')
+        repo = git.Repo.clone_from(FORK_REMOTE_URL, path)
+    else:
+        logging.info('repository exist, will reuse')
+        repo = git.Repo(path)  # type: git.Repo
+        repo.remote('origin').set_url(FORK_REMOTE_URL)
+    os.chdir(path)
+    logging.info(f'working dir {os.getcwd()}')
+    logging.info(f'Syncing origin and upstream branches {branches}')
+    if 'upstream' not in repo.remotes:
+        repo.create_remote('upstream', url=LLVM_GITHUB_URL)
+    repo.git.fetch('--all')
+    for b in branches:
+        logging.info(f'syncing branch {b}')
+        if find_commit(repo, b) is None:
+            logging.info(f'new head {b}')
+            repo.create_head(b)
+        h = repo.heads[b]
+        h.checkout()
+        repo.git.reset('--hard', f'upstream/{b}')
+        repo.git.clean('-ffxdq')
+        repo.git.push('origin', h)
+    repo.git.push('origin', '--tags')
+
+
+def find_commit(repo, rev):
+    try:
+        return repo.commit(rev)
+    except:
+        return None
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Sync LLVM fork with origina rpo.')
+    parser.add_argument('--path', type=str, help='repository path', required=True)
+    parser.add_argument('--branch', nargs='+', help='branch to sync (specify multiple to sync many branches)',
+                        required=True)
+    parser.add_argument('--log-level', type=str, default='INFO')
+    args = parser.parse_args()
+    logging.basicConfig(level=args.log_level, format='%(levelname)-7s %(message)s')
+    sync_fork(args.path, args.branch)