From 1505cfb635891acb6ebd2f5b078d5023e9e2fb5e Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Thu, 14 May 2020 12:57:38 +0200 Subject: [PATCH] build --- .../buildkite-premerge-debian/Dockerfile | 2 - .../buildkite-premerge-debian/start_agent.sh | 10 ++--- kubernetes/buildkite/agents_premerge.yaml | 2 + phabricator-proxy/main.py | 2 +- scripts/buildkite/apply_patch.sh | 18 ++++++++ ...e_pipeline.py => build_branch_pipeline.py} | 28 +++++++----- scripts/buildkite/create_branch_pipeline.py | 44 +++++++++++++++++++ scripts/phabtalk/apply_patch2.py | 19 ++++++-- 8 files changed, 102 insertions(+), 23 deletions(-) rename scripts/buildkite/{create_pipeline.py => build_branch_pipeline.py} (51%) create mode 100755 scripts/buildkite/create_branch_pipeline.py diff --git a/containers/buildkite-premerge-debian/Dockerfile b/containers/buildkite-premerge-debian/Dockerfile index c6f3957..c71cf83 100644 --- a/containers/buildkite-premerge-debian/Dockerfile +++ b/containers/buildkite-premerge-debian/Dockerfile @@ -7,8 +7,6 @@ RUN echo 'install buildkite' ;\ apt-get update ;\ apt-get install -y buildkite-agent; \ apt-get clean; - COPY *.sh /usr/local/bin/ RUN chmod og+rx /usr/local/bin/*.sh -ENV CCACHE_PATH=/mnt/disks/ssd0/ccache CMD ["start_agent.sh"] \ No newline at end of file diff --git a/containers/buildkite-premerge-debian/start_agent.sh b/containers/buildkite-premerge-debian/start_agent.sh index e85e877..3a0414c 100755 --- a/containers/buildkite-premerge-debian/start_agent.sh +++ b/containers/buildkite-premerge-debian/start_agent.sh @@ -15,12 +15,12 @@ # Buildkite installation creates 'buildkite-agent' user. USER=buildkite-agent -SSD_ROOT="/mnt/disks/ssd0" -AGENT_ROOT="${SSD_ROOT}/agent" # prepare work directory -mkdir -p "${AGENT_ROOT}" -chown -R ${USER}:${USER} "${AGENT_ROOT}" +mkdir -p "${BUILDKITE_BUILD_PATH}" +chown -R ${USER}:${USER} "${BUILDKITE_BUILD_PATH}" + +export CCACHE_PATH="${BUILDKITE_BUILD_PATH}"/ccache mkdir -p "${CCACHE_PATH}" chown -R ${USER}:${USER} "${CCACHE_PATH}" @@ -31,4 +31,4 @@ chmod 700 /var/lib/buildkite-agent/.ssh chmod 600 /var/lib/buildkite-agent/.ssh/* chown -R $USER:$USER /var/lib/buildkite-agent/.ssh -su buildkite-agent -c "buildkite-agent start --build-path=/mnt/disks/ssd0/agent" \ No newline at end of file +su buildkite-agent -c "buildkite-agent start" \ No newline at end of file diff --git a/kubernetes/buildkite/agents_premerge.yaml b/kubernetes/buildkite/agents_premerge.yaml index 6a9dd94..3d1c64d 100644 --- a/kubernetes/buildkite/agents_premerge.yaml +++ b/kubernetes/buildkite/agents_premerge.yaml @@ -47,6 +47,8 @@ spec: key: token - name: BUILDKITE_AGENT_TAGS value: "queue=premerge,os=linux" + - name: BUILDKITE_BUILD_PATH + value: "/mnt/disks/ssd0/agent" - name: CONDUIT_TOKEN valueFrom: secretKeyRef: diff --git a/phabricator-proxy/main.py b/phabricator-proxy/main.py index bb6285d..3c48349 100644 --- a/phabricator-proxy/main.py +++ b/phabricator-proxy/main.py @@ -44,7 +44,7 @@ def build(): headers = {'Authorization': f'Bearer {buildkite_api_token}'} response = requests.post( 'https://api.buildkite.com/v2/organizations/llvm-project' - '/pipelines/premerge/builds', + '/pipelines/diff-checks/builds', json=build_request, headers=headers) app.logger.info('buildkite response: %s %s', response.status_code, response.text) diff --git a/scripts/buildkite/apply_patch.sh b/scripts/buildkite/apply_patch.sh index afdd8c2..20407e0 100755 --- a/scripts/buildkite/apply_patch.sh +++ b/scripts/buildkite/apply_patch.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Copyright 2020 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Scripts that use secrets has to be a separate files, not inlined in pipeline: +# https://buildkite.com/docs/pipelines/secrets#anti-pattern-referencing-secrets-in-your-pipeline-yaml + scripts/phabtalk/apply_patch2.py $ph_buildable_diff \ + --path "${BUILDKITE_BUILD_PATH}"/llvm-project \ --token $CONDUIT_TOKEN \ --url $PHABRICATOR_HOST \ --comment-file apply_patch.txt \ diff --git a/scripts/buildkite/create_pipeline.py b/scripts/buildkite/build_branch_pipeline.py similarity index 51% rename from scripts/buildkite/create_pipeline.py rename to scripts/buildkite/build_branch_pipeline.py index f35d0ab..3226d22 100755 --- a/scripts/buildkite/create_pipeline.py +++ b/scripts/buildkite/build_branch_pipeline.py @@ -14,17 +14,23 @@ # limitations under the License. import os +import yaml if __name__ == '__main__': - script_branch = os.getenv("PREMERGE_SCRIPTS_BRANCH", "master") + script_branch = os.getenv("scripts_branch", "master") queue = os.getenv("BUILDKITE_AGENT_META_DATA_QUEUE", "default") - print(f""" - steps: - - label: "build" - commands: - - "git clone git@github.com:llvm-premerge-tests/llvm-project.git" - - "scripts/buildkite/apply_patch.sh" - agents: - queue: "{queue}" - os: "linux" - """) + diff_id = os.getenv("ph_buildable_diff", "") + steps = [] + # SCRIPT_DIR is defined in buildkite pipeline step. + linux_buld_step = { + 'label': 'build linux', + 'key': 'build-linux', + 'commands': [ + '${SCRIPT_DIR}/run_cmake.py detect', + '${SCRIPT_DIR}/run_ninja.py all', + '${SCRIPT_DIR}/run_ninja.py check-all', + '${SCRIPT_DIR}/lint.sh HEAD~1 ./'], + 'agents': {'queue': queue, 'os': 'linux'} + } + steps.append(linux_buld_step) + print(yaml.dump({'steps': steps})) \ No newline at end of file diff --git a/scripts/buildkite/create_branch_pipeline.py b/scripts/buildkite/create_branch_pipeline.py new file mode 100755 index 0000000..d7dd1d2 --- /dev/null +++ b/scripts/buildkite/create_branch_pipeline.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# Copyright 2020 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import yaml + +if __name__ == '__main__': + queue = os.getenv("BUILDKITE_AGENT_META_DATA_QUEUE", "default") + diff_id = os.getenv("ph_buildable_diff") + steps = [] + create_branch_step = { + 'label': 'create branch', + 'key': 'create-branch', + 'commands': ['scripts/buildkite/apply_patch.sh'], + 'agents': {'queue': queue, 'os': 'linux'} + } + run_build_step = { + 'trigger': 'premerge-checks', + 'label': ':rocket: build', + 'async': False, + 'depends_on': 'create-branch', + 'build': { + 'branch': f'phab-diff-{diff_id}', + 'env': {'scripts_branch': '${BUILDKITE_BRANCH}'}, + }, + } + for e in os.environ: + if e.startswith('ph_'): + run_build_step['build']['env'][e] = os.getenv(e) + steps.append(create_branch_step) + steps.append(run_build_step) + print(yaml.dump({'steps': steps})) diff --git a/scripts/phabtalk/apply_patch2.py b/scripts/phabtalk/apply_patch2.py index 8336071..0f5259d 100755 --- a/scripts/phabtalk/apply_patch2.py +++ b/scripts/phabtalk/apply_patch2.py @@ -30,6 +30,7 @@ from git import Repo, BadName, GitCommandError # FIXME: maybe move to config file """URL of upstream LLVM repository.""" LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project' +FORK_REMOTE_URL = 'ssh://git@github.com/llvm-premerge-tests/llvm-project' """How far back the script searches in the git history to find Revisions that have already landed. """ @@ -49,10 +50,10 @@ class ApplyPatch: Once this class has applied all dependencies, it will apply the diff itself. This script must be called from the root folder of a local checkout of - https://github.com/llvm/llvm-project + https://github.com/llvm/llvm-project or given a path to clone into. """ - def __init__(self, diff_id: int, comment_file_path: str, token: str, url: str, git_hash: str, + def __init__(self, path: str, diff_id: int, comment_file_path: str, token: str, url: str, git_hash: str, push_branch: bool = False): self.comment_file_path = comment_file_path self.push_branch = push_branch # type: bool @@ -65,7 +66,16 @@ class ApplyPatch: self.phab = self._create_phab() self.base_revision = git_hash # type: Optional[str] self.msg = [] # type: List[str] - self.repo = Repo(os.getcwd()) # type: Repo + + if not os.path.isdir(path): + print(f'{path} does not exist, cloning repository') + # TODO: progress of clonning + self.repo = Repo.clone_from(FORK_REMOTE_URL, path) + else: + print('repository exist, will reuse') + self.repo = Repo(path) # type: Repo + os.chdir(path) + print('working dir', os.getcwd()) @property def branch_name(self): @@ -308,6 +318,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description='Apply Phabricator patch to working directory.') parser.add_argument('diff_id', type=int) # TODO: instead of --comment-file use stdout / stderr. + parser.add_argument('--path', type=str, help='repository path', default=os.getcwd()) parser.add_argument('--comment-file', type=str, dest='comment_file_path', default=None) parser.add_argument('--token', type=str, default=None, help='Conduit API token') parser.add_argument('--url', type=str, default=None, help='Phabricator URL') @@ -316,5 +327,5 @@ if __name__ == "__main__": parser.add_argument('--push-branch', action='store_true', dest='push_branch', help='choose if branch shall be pushed to origin') args = parser.parse_args() - patcher = ApplyPatch(args.diff_id, args.comment_file_path, args.token, args.url, args.commit, args.push_branch) + patcher = ApplyPatch(args.path, args.diff_id, args.comment_file_path, args.token, args.url, args.commit, args.push_branch) patcher.run()