Merge pull request #183 from google/bk-build

build on buildkite

containers/buildkite-premerge-debian/Dockerfile

@@ -7,8 +7,6 @@ RUN echo 'install buildkite' ;\
     apt-get update ;\
     apt-get install -y buildkite-agent; \
     apt-get clean;
-
 COPY *.sh /usr/local/bin/
 RUN chmod og+rx /usr/local/bin/*.sh
-ENV CCACHE_PATH=/mnt/disks/ssd0/ccache
 CMD ["start_agent.sh"]

containers/buildkite-premerge-debian/start_agent.sh

@@ -15,12 +15,12 @@
 
 # Buildkite installation creates 'buildkite-agent' user.
 USER=buildkite-agent
-SSD_ROOT="/mnt/disks/ssd0"
-AGENT_ROOT="${SSD_ROOT}/agent"
 
 # prepare work directory
-mkdir -p "${AGENT_ROOT}"
+mkdir -p "${BUILDKITE_BUILD_PATH}"
-chown -R ${USER}:${USER} "${AGENT_ROOT}"
+chown -R ${USER}:${USER} "${BUILDKITE_BUILD_PATH}"
+
+export CCACHE_PATH="${BUILDKITE_BUILD_PATH}"/ccache
 mkdir -p "${CCACHE_PATH}"
 chown -R ${USER}:${USER} "${CCACHE_PATH}"
 
@@ -31,4 +31,4 @@ chmod 700 /var/lib/buildkite-agent/.ssh
 chmod 600 /var/lib/buildkite-agent/.ssh/*
 chown -R $USER:$USER /var/lib/buildkite-agent/.ssh
 
-su buildkite-agent -c "buildkite-agent start --build-path=/mnt/disks/ssd0/agent"
+su buildkite-agent -c "buildkite-agent start"

kubernetes/buildkite/agents_premerge.yaml

@@ -47,6 +47,8 @@ spec:
               key: token
         - name: BUILDKITE_AGENT_TAGS
           value: "queue=premerge,os=linux"
+        - name: BUILDKITE_BUILD_PATH
+          value: "/mnt/disks/ssd0/agent"
         - name: CONDUIT_TOKEN
           valueFrom:
             secretKeyRef:

phabricator-proxy/main.py

@@ -44,7 +44,7 @@ def build():
         headers = {'Authorization': f'Bearer {buildkite_api_token}'}
         response = requests.post(
             'https://api.buildkite.com/v2/organizations/llvm-project'
-            '/pipelines/premerge/builds',
+            '/pipelines/diff-checks/builds',
             json=build_request,
             headers=headers)
         app.logger.info('buildkite response: %s %s', response.status_code, response.text)

scripts/buildkite/apply_patch.sh

@@ -1,5 +1,23 @@
 #!/usr/bin/env bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Scripts that use secrets has to be a separate files, not inlined in pipeline:
+# https://buildkite.com/docs/pipelines/secrets#anti-pattern-referencing-secrets-in-your-pipeline-yaml
+
 scripts/phabtalk/apply_patch2.py $ph_buildable_diff \
+  --path "${BUILDKITE_BUILD_PATH}"/llvm-project \
   --token $CONDUIT_TOKEN \
   --url $PHABRICATOR_HOST \
   --comment-file apply_patch.txt \

scripts/buildkite/build_branch_pipeline.py

@@ -14,17 +14,23 @@
 # limitations under the License.
 
 import os
+import yaml
 
 if __name__ == '__main__':
-    script_branch = os.getenv("PREMERGE_SCRIPTS_BRANCH", "master")
+    script_branch = os.getenv("scripts_branch", "master")
     queue = os.getenv("BUILDKITE_AGENT_META_DATA_QUEUE", "default")
-    print(f"""
+    diff_id = os.getenv("ph_buildable_diff", "")
-  steps:
+    steps = []
-    - label: "build"
+    # SCRIPT_DIR is defined in buildkite pipeline step.
-      commands:
+    linux_buld_step = {
-      - "git clone git@github.com:llvm-premerge-tests/llvm-project.git"
+            'label': 'build linux',
-      - "scripts/buildkite/apply_patch.sh" 
+            'key': 'build-linux',
-      agents:
+            'commands': [
-          queue: "{queue}"
+                    '${SCRIPT_DIR}/run_cmake.py detect',
-          os: "linux"
+                    '${SCRIPT_DIR}/run_ninja.py all',
-    """)
+                    '${SCRIPT_DIR}/run_ninja.py check-all',
+                    '${SCRIPT_DIR}/lint.sh HEAD~1 ./'],
+            'agents': {'queue': queue, 'os': 'linux'}
+    }
+    steps.append(linux_buld_step)
+    print(yaml.dump({'steps': steps}))

scripts/buildkite/create_branch_pipeline.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# Copyright 2020 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import yaml
+
+if __name__ == '__main__':
+    queue = os.getenv("BUILDKITE_AGENT_META_DATA_QUEUE", "default")
+    diff_id = os.getenv("ph_buildable_diff")
+    steps = []
+    create_branch_step = {
+            'label': 'create branch',
+            'key': 'create-branch',
+            'commands': ['scripts/buildkite/apply_patch.sh'],
+            'agents': {'queue': queue, 'os': 'linux'}
+    }
+    run_build_step = {
+            'trigger': 'premerge-checks',
+            'label': ':rocket: build',
+            'async': False,
+            'depends_on': 'create-branch',
+            'build': {
+                    'branch': f'phab-diff-{diff_id}',
+                    'env': {'scripts_branch': '${BUILDKITE_BRANCH}'},
+            },
+    }
+    for e in os.environ:
+        if e.startswith('ph_'):
+            run_build_step['build']['env'][e] = os.getenv(e)
+    steps.append(create_branch_step)
+    steps.append(run_build_step)
+    print(yaml.dump({'steps': steps}))

scripts/phabtalk/apply_patch2.py

@@ -30,6 +30,7 @@ from git import Repo, BadName, GitCommandError
 # FIXME: maybe move to config file
 """URL of upstream LLVM repository."""
 LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project'
+FORK_REMOTE_URL = 'ssh://git@github.com/llvm-premerge-tests/llvm-project'
 
 """How far back the script searches in the git history to find Revisions that
 have already landed. """
@@ -49,10 +50,10 @@ class ApplyPatch:
     Once this class has applied all dependencies, it will apply the diff itself.
 
     This script must be called from the root folder of a local checkout of 
-    https://github.com/llvm/llvm-project
+    https://github.com/llvm/llvm-project or given a path to clone into.
     """
 
-    def __init__(self, diff_id: int, comment_file_path: str, token: str, url: str, git_hash: str,
+    def __init__(self, path: str, diff_id: int, comment_file_path: str, token: str, url: str, git_hash: str,
                  push_branch: bool = False):
         self.comment_file_path = comment_file_path
         self.push_branch = push_branch  # type: bool
@@ -65,7 +66,16 @@ class ApplyPatch:
         self.phab = self._create_phab()
         self.base_revision = git_hash  # type: Optional[str]
         self.msg = []  # type: List[str]
-        self.repo = Repo(os.getcwd())  # type: Repo
+
+        if not os.path.isdir(path):
+            print(f'{path} does not exist, cloning repository')
+            # TODO: progress of clonning
+            self.repo = Repo.clone_from(FORK_REMOTE_URL, path)
+        else:
+            print('repository exist, will reuse')
+            self.repo = Repo(path)  # type: Repo
+        os.chdir(path)
+        print('working dir', os.getcwd())
 
     @property
     def branch_name(self):
@@ -308,6 +318,7 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='Apply Phabricator patch to working directory.')
     parser.add_argument('diff_id', type=int)
     # TODO: instead of --comment-file use stdout / stderr.
+    parser.add_argument('--path', type=str, help='repository path', default=os.getcwd())
     parser.add_argument('--comment-file', type=str, dest='comment_file_path', default=None)
     parser.add_argument('--token', type=str, default=None, help='Conduit API token')
     parser.add_argument('--url', type=str, default=None, help='Phabricator URL')
@@ -316,5 +327,5 @@ if __name__ == "__main__":
     parser.add_argument('--push-branch', action='store_true', dest='push_branch',
                         help='choose if branch shall be pushed to origin')
     args = parser.parse_args()
-    patcher = ApplyPatch(args.diff_id, args.comment_file_path, args.token, args.url, args.commit, args.push_branch)
+    patcher = ApplyPatch(args.path, args.diff_id, args.comment_file_path, args.token, args.url, args.commit, args.push_branch)
     patcher.run()