1
0
Fork 0
This commit is contained in:
Mikhail Goncharov 2020-05-14 12:57:38 +02:00
parent 097df96734
commit 1505cfb635
8 changed files with 102 additions and 23 deletions

View file

@ -7,8 +7,6 @@ RUN echo 'install buildkite' ;\
apt-get update ;\
apt-get install -y buildkite-agent; \
apt-get clean;
COPY *.sh /usr/local/bin/
RUN chmod og+rx /usr/local/bin/*.sh
ENV CCACHE_PATH=/mnt/disks/ssd0/ccache
CMD ["start_agent.sh"]

View file

@ -15,12 +15,12 @@
# Buildkite installation creates 'buildkite-agent' user.
USER=buildkite-agent
SSD_ROOT="/mnt/disks/ssd0"
AGENT_ROOT="${SSD_ROOT}/agent"
# prepare work directory
mkdir -p "${AGENT_ROOT}"
chown -R ${USER}:${USER} "${AGENT_ROOT}"
mkdir -p "${BUILDKITE_BUILD_PATH}"
chown -R ${USER}:${USER} "${BUILDKITE_BUILD_PATH}"
export CCACHE_PATH="${BUILDKITE_BUILD_PATH}"/ccache
mkdir -p "${CCACHE_PATH}"
chown -R ${USER}:${USER} "${CCACHE_PATH}"
@ -31,4 +31,4 @@ chmod 700 /var/lib/buildkite-agent/.ssh
chmod 600 /var/lib/buildkite-agent/.ssh/*
chown -R $USER:$USER /var/lib/buildkite-agent/.ssh
su buildkite-agent -c "buildkite-agent start --build-path=/mnt/disks/ssd0/agent"
su buildkite-agent -c "buildkite-agent start"

View file

@ -47,6 +47,8 @@ spec:
key: token
- name: BUILDKITE_AGENT_TAGS
value: "queue=premerge,os=linux"
- name: BUILDKITE_BUILD_PATH
value: "/mnt/disks/ssd0/agent"
- name: CONDUIT_TOKEN
valueFrom:
secretKeyRef:

View file

@ -44,7 +44,7 @@ def build():
headers = {'Authorization': f'Bearer {buildkite_api_token}'}
response = requests.post(
'https://api.buildkite.com/v2/organizations/llvm-project'
'/pipelines/premerge/builds',
'/pipelines/diff-checks/builds',
json=build_request,
headers=headers)
app.logger.info('buildkite response: %s %s', response.status_code, response.text)

View file

@ -1,5 +1,23 @@
#!/usr/bin/env bash
# Copyright 2020 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Scripts that use secrets has to be a separate files, not inlined in pipeline:
# https://buildkite.com/docs/pipelines/secrets#anti-pattern-referencing-secrets-in-your-pipeline-yaml
scripts/phabtalk/apply_patch2.py $ph_buildable_diff \
--path "${BUILDKITE_BUILD_PATH}"/llvm-project \
--token $CONDUIT_TOKEN \
--url $PHABRICATOR_HOST \
--comment-file apply_patch.txt \

View file

@ -14,17 +14,23 @@
# limitations under the License.
import os
import yaml
if __name__ == '__main__':
script_branch = os.getenv("PREMERGE_SCRIPTS_BRANCH", "master")
script_branch = os.getenv("scripts_branch", "master")
queue = os.getenv("BUILDKITE_AGENT_META_DATA_QUEUE", "default")
print(f"""
steps:
- label: "build"
commands:
- "git clone git@github.com:llvm-premerge-tests/llvm-project.git"
- "scripts/buildkite/apply_patch.sh"
agents:
queue: "{queue}"
os: "linux"
""")
diff_id = os.getenv("ph_buildable_diff", "")
steps = []
# SCRIPT_DIR is defined in buildkite pipeline step.
linux_buld_step = {
'label': 'build linux',
'key': 'build-linux',
'commands': [
'${SCRIPT_DIR}/run_cmake.py detect',
'${SCRIPT_DIR}/run_ninja.py all',
'${SCRIPT_DIR}/run_ninja.py check-all',
'${SCRIPT_DIR}/lint.sh HEAD~1 ./'],
'agents': {'queue': queue, 'os': 'linux'}
}
steps.append(linux_buld_step)
print(yaml.dump({'steps': steps}))

View file

@ -0,0 +1,44 @@
#!/usr/bin/env python3
# Copyright 2020 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import yaml
if __name__ == '__main__':
queue = os.getenv("BUILDKITE_AGENT_META_DATA_QUEUE", "default")
diff_id = os.getenv("ph_buildable_diff")
steps = []
create_branch_step = {
'label': 'create branch',
'key': 'create-branch',
'commands': ['scripts/buildkite/apply_patch.sh'],
'agents': {'queue': queue, 'os': 'linux'}
}
run_build_step = {
'trigger': 'premerge-checks',
'label': ':rocket: build',
'async': False,
'depends_on': 'create-branch',
'build': {
'branch': f'phab-diff-{diff_id}',
'env': {'scripts_branch': '${BUILDKITE_BRANCH}'},
},
}
for e in os.environ:
if e.startswith('ph_'):
run_build_step['build']['env'][e] = os.getenv(e)
steps.append(create_branch_step)
steps.append(run_build_step)
print(yaml.dump({'steps': steps}))

View file

@ -30,6 +30,7 @@ from git import Repo, BadName, GitCommandError
# FIXME: maybe move to config file
"""URL of upstream LLVM repository."""
LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project'
FORK_REMOTE_URL = 'ssh://git@github.com/llvm-premerge-tests/llvm-project'
"""How far back the script searches in the git history to find Revisions that
have already landed. """
@ -49,10 +50,10 @@ class ApplyPatch:
Once this class has applied all dependencies, it will apply the diff itself.
This script must be called from the root folder of a local checkout of
https://github.com/llvm/llvm-project
https://github.com/llvm/llvm-project or given a path to clone into.
"""
def __init__(self, diff_id: int, comment_file_path: str, token: str, url: str, git_hash: str,
def __init__(self, path: str, diff_id: int, comment_file_path: str, token: str, url: str, git_hash: str,
push_branch: bool = False):
self.comment_file_path = comment_file_path
self.push_branch = push_branch # type: bool
@ -65,7 +66,16 @@ class ApplyPatch:
self.phab = self._create_phab()
self.base_revision = git_hash # type: Optional[str]
self.msg = [] # type: List[str]
self.repo = Repo(os.getcwd()) # type: Repo
if not os.path.isdir(path):
print(f'{path} does not exist, cloning repository')
# TODO: progress of clonning
self.repo = Repo.clone_from(FORK_REMOTE_URL, path)
else:
print('repository exist, will reuse')
self.repo = Repo(path) # type: Repo
os.chdir(path)
print('working dir', os.getcwd())
@property
def branch_name(self):
@ -308,6 +318,7 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Apply Phabricator patch to working directory.')
parser.add_argument('diff_id', type=int)
# TODO: instead of --comment-file use stdout / stderr.
parser.add_argument('--path', type=str, help='repository path', default=os.getcwd())
parser.add_argument('--comment-file', type=str, dest='comment_file_path', default=None)
parser.add_argument('--token', type=str, default=None, help='Conduit API token')
parser.add_argument('--url', type=str, default=None, help='Phabricator URL')
@ -316,5 +327,5 @@ if __name__ == "__main__":
parser.add_argument('--push-branch', action='store_true', dest='push_branch',
help='choose if branch shall be pushed to origin')
args = parser.parse_args()
patcher = ApplyPatch(args.diff_id, args.comment_file_path, args.token, args.url, args.commit, args.push_branch)
patcher = ApplyPatch(args.path, args.diff_id, args.comment_file_path, args.token, args.url, args.commit, args.push_branch)
patcher.run()