Run bazel builds in premerge

if user is a member of "bazel_build" https://reviews.llvm.org/project/view/107/
or modified /utils/bazel/*

For #328

containers/buildkite-premerge-debian/post-checkout

@@ -16,7 +16,8 @@
 
 set -eu
 
-# Run prune as git will start to complain about
+# Run prune as git will start to complain "warning: There are too many unreachable loose objects; run 'git prune' to remove them.".
+# That happens as we routinely drop old branches.
 git prune
 if [ -f ".git/gc.log" ]; then
   echo ".git/gc.log exist"

docs/playbooks.md

@@ -192,7 +192,18 @@ Most commonly used are:
 - `ph_log_level` ("DEBUG", "INFO", "WARNING" (default) or "ERROR"): log level for build scripts. 
 - `ph_linux_agents`, `ph_windows_agents`: custom JSON constraints on agents. For example, you might put one machine to a custom queue if it's errornous and send jobs to it with `ph_windows_agents={"queue": "custom"}`.
 - `ph_skip_linux`, `ph_skip_windows` (if set to any value): skip build on this OS.
-- `ph_skip_generated`: don't run custom steps generated from within llvm-project. 
+- `ph_skip_generated`: don't run custom steps generated from within llvm-project.
+
+While trying a new patch for premerge scripts it's typical to start a new build by copying "ph_"
+env variables from one of the recent builds and appending
+
+```shell
+ph_dry_run_report=yes
+ph_skip_windows=yes
+ph_skip_generated=yes
+ph_scripts_refspec="<branch name>"
+ph_log_level=DEBUG
+```
 
 ## Update HTTP auth credentials
 

scripts/choose_projects.py

@@ -122,7 +122,8 @@ class ChooseProjects:
 
     @staticmethod
     def get_changed_files(patch_str: str = None) -> Set[str]:
-        """get list of changed files from the patch or from STDIN."""
+        """get list of changed files from the patch or from STDIN.
+        e.g. ['compiler-rt/lib/tsan/CMakeLists.txt']"""
         if patch_str is None:
             patch_str = sys.stdin
         patch = PatchSet(patch_str)

scripts/patch_diff.py

@@ -219,7 +219,7 @@ class ApplyPatch:
 
     @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
     def get_diff(self, diff_id: int):
-        """Get a diff from Phabricator based on it's diff id."""
+        """Get a diff from Phabricator based on its diff id."""
         return self.phab.differential.getdiff(diff_id=diff_id)
 
     @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)

scripts/phabtalk/phabtalk.py

@@ -23,11 +23,13 @@ import argparse
 
 import backoff
 from phabricator import Phabricator
+from benedict import benedict
 
 
 class PhabTalk:
     """Talk to Phabricator to upload build results.
        See https://secure.phabricator.com/conduit/method/harbormaster.sendmessage/
+       You might want to use it as it provides retries on most of the calls.
     """
 
     def __init__(self, token: Optional[str], host: Optional[str] = 'https://reviews.llvm.org/api/',
@@ -47,6 +49,11 @@ class PhabTalk:
         result = self._phab.differential.querydiffs(ids=[diff])
         return 'D' + result[diff]['revisionID']
 
+    @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
+    def get_diff(self, diff_id: int):
+        """Get a diff from Phabricator based on its diff id."""
+        return self._phab.differential.getdiff(diff_id=diff_id)
+
     def comment_on_diff(self, diff_id: str, text: str):
         """Add a comment to a differential based on the diff_id"""
         logging.info('Sending comment to diff {}:'.format(diff_id))
@@ -144,6 +151,27 @@ class PhabTalk:
             return
         self.create_artifact(phid, str(uuid.uuid4()), 'uri', {'uri': url, 'ui.external': True, 'name': name})
 
+    @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
+    def user_projects(self, user_phid: str) -> List[str]:
+        """Returns slugs of all projects user has a membership."""
+        projects = benedict(self._phab.project.search(constraints={'members': [user_phid]}))
+        slugs = []
+        for p in projects.get('data', []):
+            slug = benedict(p).get('fields.slug')
+            if slug:
+                slugs.append(p['fields']['slug'])
+        return slugs
+
+    @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
+    def get_revision(self, revision_id: int):
+        """Get a revision from Phabricator based on its revision id."""
+        return self._phab.differential.query(ids=[revision_id])[0]
+
+    @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
+    def get_diff(self, diff_id: int):
+        """Get a diff from Phabricator based on its diff id."""
+        return self._phab.differential.getdiff(diff_id=diff_id)
+
 
 class Step:
     def __init__(self, name=''):

scripts/pipeline_create_branch.py

@@ -14,19 +14,31 @@
 # limitations under the License.
 
 import os
+import sys
+
 import yaml
 import logging
 from buildkite_utils import set_metadata, BuildkiteApi
-
+from phabtalk.phabtalk import PhabTalk
 
 if __name__ == '__main__':
     diff_id = os.getenv("ph_buildable_diff")
+    revision_id = os.getenv("ph_buildable_revision")
     log_level = os.getenv('ph_log_level', 'INFO')
     base_commit = os.getenv('ph_base_commit', 'auto')
     run_build = os.getenv('ph_skip_build') is None
     trigger = os.getenv('ph_trigger_pipeline')
     logging.basicConfig(level=log_level, format='%(levelname)-7s %(message)s')
 
+    phabtalk = PhabTalk(os.getenv('CONDUIT_TOKEN'), dry_run_updates=(os.getenv('ph_dry_run_report') is not None))
+    rev = phabtalk.get_revision(int(revision_id))
+    user_id = rev.get('authorPHID')
+    logging.debug(f'authorPHID {user_id}')
+    if user_id is None:
+        logging.error('cannot find author of the revision')
+        sys.exit(1)
+    projects = phabtalk.user_projects(user_id)
+    logging.info(f'user projects: {", ".join(projects)}')
     # Cancel any existing builds.
     # Do this before setting own 'ph_buildable_revision'.
     try:
@@ -43,8 +55,17 @@ if __name__ == '__main__':
     if trigger is None:
         trigger = 'premerge-checks'
 
-    steps = []
+    env = {
-    steps.append({
+        'ph_scripts_refspec': '${BUILDKITE_BRANCH}',
+        'ph_user_project_slugs': ",".join(projects),
+        # TODO: those two are for "apply_patch.sh". Maybe just look for "ph_" env in patch_diff.py?
+        'LOG_LEVEL': log_level,
+        'BASE_COMMIT': base_commit,
+    }
+    for e in os.environ:
+        if e.startswith('ph_'):
+            env[e] = os.getenv(e)
+    steps = [{
         'label': 'create branch',
         'key': 'create-branch',
         'commands': [
@@ -53,11 +74,8 @@ if __name__ == '__main__':
         ],
         'agents': {'queue': 'service'},
         'timeout_in_minutes': 20,
-        'env': {
+        'env': env
-            'LOG_LEVEL': log_level,
+    }]
-            'BASE_COMMIT': base_commit,
-        }
-    })
     if run_build:
         trigger_build_step = {
             'trigger': trigger,
@@ -66,14 +84,8 @@ if __name__ == '__main__':
             'depends_on': 'create-branch',
             'build': {
                 'branch': f'phab-diff-{diff_id}',
-                'env': {},
+                'env': env,
             },
         }
-        for e in os.environ:
-            if e.startswith('ph_'):
-                trigger_build_step['build']['env'][e] = os.getenv(e)
-        # Set scripts source from the current build if it's not yet defined.
-        if 'ph_scripts_refspec' not in trigger_build_step['build']['env']:
-            trigger_build_step['build']['env']['ph_scripts_refspec'] = '${BUILDKITE_BRANCH}'
         steps.append(trigger_build_step)
     print(yaml.dump({'steps': steps}))

scripts/pipeline_premerge.py

@@ -19,7 +19,7 @@ import os
 from buildkite_utils import annotate, feedback_url, set_metadata
 from choose_projects import ChooseProjects
 import git
-from steps import generic_linux, generic_windows, from_shell_output, checkout_scripts
+from steps import generic_linux, generic_windows, from_shell_output, checkout_scripts, bazel
 import yaml
 
 steps_generators = [
@@ -73,6 +73,7 @@ if __name__ == '__main__':
     if os.getenv('ph_skip_generated') is None:
         for gen in steps_generators:
             steps.extend(from_shell_output(gen))
+    steps.extend(bazel(modified_files))
 
     if phid is None:
         logging.warning('ph_target_phid is not specified. Skipping "Report" step')

scripts/steps.py

@@ -17,7 +17,7 @@ import io
 import json
 import logging
 import os
-from typing import List
+from typing import List, Set
 
 from exec_utils import watch_shell
 import yaml
@@ -75,6 +75,40 @@ def generic_linux(projects: str, check_diff: bool) -> List:
     return [linux_buld_step]
 
 
+def bazel(modified_files: Set[str]) -> List:
+    if os.getenv('ph_skip_bazel') is not None:
+        logging.info('bazel build is skipped as "ph_skip_bazel" is set')
+        return []
+    updated_build = any(s.startswith('utils/bazel/') for s in modified_files)
+    if updated_build:
+        logging.info('files in utils/bazel/ modified, will trigger bazel build')
+    else:
+        user_projects = os.getenv('ph_user_project_slugs', '').split(',')
+        if 'bazel_build' not in user_projects:
+            logging.info('bazel build is skipped as "bazel_build" is not listed in user projects and no files in '
+                         'utils/bazel/ are modified')
+            return []
+    agents = {'queue': 'llvm-bazel'}
+    t = os.getenv('ph_bazel_agents')
+    if t is not None:
+        agents = json.loads(t)
+    return [{
+        'label': ':bazel: bazel',
+        'key': 'bazel',
+        'commands': [
+            'set -eu',
+            'cd utils/bazel',
+            'bazel query //... + @llvm-project//... | xargs bazel test --config=generic_clang --config=rbe --test_output=errors --test_tag_filters=-nobuildkite --build_tag_filters=-nobuildkite',
+        ],
+        'agents': agents,
+        'timeout_in_minutes': 120,
+        'retry': {'automatic': [
+            {'exit_status': -1, 'limit': 2},  # Agent lost
+            {'exit_status': 255, 'limit': 2},  # Forced agent shutdown
+        ]},
+    }]
+
+
 def generic_windows(projects: str) -> List:
     if os.getenv('ph_skip_windows') is not None:
         return []

scripts/summary.py

@@ -75,7 +75,7 @@ if __name__ == '__main__':
         for i, job in enumerate(build.get('jobs', [])):
             job = benedict(job)
             job_web_url = job.get('web_url', os.getenv('BUILDKITE_BUILD_URL', ''))
-            logging.info(f'{job.get("id")} state {job.get("state")}')
+            logging.info(f'{job.get("id")} {job.get("name")} state {job.get("state")}')
             job_state = job.get('state')
             if job.get('type') == 'waiter':
                 continue