build
This commit is contained in:
parent
097df96734
commit
1505cfb635
8 changed files with 102 additions and 23 deletions
|
@ -7,8 +7,6 @@ RUN echo 'install buildkite' ;\
|
|||
apt-get update ;\
|
||||
apt-get install -y buildkite-agent; \
|
||||
apt-get clean;
|
||||
|
||||
COPY *.sh /usr/local/bin/
|
||||
RUN chmod og+rx /usr/local/bin/*.sh
|
||||
ENV CCACHE_PATH=/mnt/disks/ssd0/ccache
|
||||
CMD ["start_agent.sh"]
|
|
@ -15,12 +15,12 @@
|
|||
|
||||
# Buildkite installation creates 'buildkite-agent' user.
|
||||
USER=buildkite-agent
|
||||
SSD_ROOT="/mnt/disks/ssd0"
|
||||
AGENT_ROOT="${SSD_ROOT}/agent"
|
||||
|
||||
# prepare work directory
|
||||
mkdir -p "${AGENT_ROOT}"
|
||||
chown -R ${USER}:${USER} "${AGENT_ROOT}"
|
||||
mkdir -p "${BUILDKITE_BUILD_PATH}"
|
||||
chown -R ${USER}:${USER} "${BUILDKITE_BUILD_PATH}"
|
||||
|
||||
export CCACHE_PATH="${BUILDKITE_BUILD_PATH}"/ccache
|
||||
mkdir -p "${CCACHE_PATH}"
|
||||
chown -R ${USER}:${USER} "${CCACHE_PATH}"
|
||||
|
||||
|
@ -31,4 +31,4 @@ chmod 700 /var/lib/buildkite-agent/.ssh
|
|||
chmod 600 /var/lib/buildkite-agent/.ssh/*
|
||||
chown -R $USER:$USER /var/lib/buildkite-agent/.ssh
|
||||
|
||||
su buildkite-agent -c "buildkite-agent start --build-path=/mnt/disks/ssd0/agent"
|
||||
su buildkite-agent -c "buildkite-agent start"
|
|
@ -47,6 +47,8 @@ spec:
|
|||
key: token
|
||||
- name: BUILDKITE_AGENT_TAGS
|
||||
value: "queue=premerge,os=linux"
|
||||
- name: BUILDKITE_BUILD_PATH
|
||||
value: "/mnt/disks/ssd0/agent"
|
||||
- name: CONDUIT_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
|
|
|
@ -44,7 +44,7 @@ def build():
|
|||
headers = {'Authorization': f'Bearer {buildkite_api_token}'}
|
||||
response = requests.post(
|
||||
'https://api.buildkite.com/v2/organizations/llvm-project'
|
||||
'/pipelines/premerge/builds',
|
||||
'/pipelines/diff-checks/builds',
|
||||
json=build_request,
|
||||
headers=headers)
|
||||
app.logger.info('buildkite response: %s %s', response.status_code, response.text)
|
||||
|
|
|
@ -1,5 +1,23 @@
|
|||
#!/usr/bin/env bash
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://llvm.org/LICENSE.txt
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Scripts that use secrets has to be a separate files, not inlined in pipeline:
|
||||
# https://buildkite.com/docs/pipelines/secrets#anti-pattern-referencing-secrets-in-your-pipeline-yaml
|
||||
|
||||
scripts/phabtalk/apply_patch2.py $ph_buildable_diff \
|
||||
--path "${BUILDKITE_BUILD_PATH}"/llvm-project \
|
||||
--token $CONDUIT_TOKEN \
|
||||
--url $PHABRICATOR_HOST \
|
||||
--comment-file apply_patch.txt \
|
||||
|
|
|
@ -14,17 +14,23 @@
|
|||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import yaml
|
||||
|
||||
if __name__ == '__main__':
|
||||
script_branch = os.getenv("PREMERGE_SCRIPTS_BRANCH", "master")
|
||||
script_branch = os.getenv("scripts_branch", "master")
|
||||
queue = os.getenv("BUILDKITE_AGENT_META_DATA_QUEUE", "default")
|
||||
print(f"""
|
||||
steps:
|
||||
- label: "build"
|
||||
commands:
|
||||
- "git clone git@github.com:llvm-premerge-tests/llvm-project.git"
|
||||
- "scripts/buildkite/apply_patch.sh"
|
||||
agents:
|
||||
queue: "{queue}"
|
||||
os: "linux"
|
||||
""")
|
||||
diff_id = os.getenv("ph_buildable_diff", "")
|
||||
steps = []
|
||||
# SCRIPT_DIR is defined in buildkite pipeline step.
|
||||
linux_buld_step = {
|
||||
'label': 'build linux',
|
||||
'key': 'build-linux',
|
||||
'commands': [
|
||||
'${SCRIPT_DIR}/run_cmake.py detect',
|
||||
'${SCRIPT_DIR}/run_ninja.py all',
|
||||
'${SCRIPT_DIR}/run_ninja.py check-all',
|
||||
'${SCRIPT_DIR}/lint.sh HEAD~1 ./'],
|
||||
'agents': {'queue': queue, 'os': 'linux'}
|
||||
}
|
||||
steps.append(linux_buld_step)
|
||||
print(yaml.dump({'steps': steps}))
|
44
scripts/buildkite/create_branch_pipeline.py
Executable file
44
scripts/buildkite/create_branch_pipeline.py
Executable file
|
@ -0,0 +1,44 @@
|
|||
#!/usr/bin/env python3
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://llvm.org/LICENSE.txt
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import yaml
|
||||
|
||||
if __name__ == '__main__':
|
||||
queue = os.getenv("BUILDKITE_AGENT_META_DATA_QUEUE", "default")
|
||||
diff_id = os.getenv("ph_buildable_diff")
|
||||
steps = []
|
||||
create_branch_step = {
|
||||
'label': 'create branch',
|
||||
'key': 'create-branch',
|
||||
'commands': ['scripts/buildkite/apply_patch.sh'],
|
||||
'agents': {'queue': queue, 'os': 'linux'}
|
||||
}
|
||||
run_build_step = {
|
||||
'trigger': 'premerge-checks',
|
||||
'label': ':rocket: build',
|
||||
'async': False,
|
||||
'depends_on': 'create-branch',
|
||||
'build': {
|
||||
'branch': f'phab-diff-{diff_id}',
|
||||
'env': {'scripts_branch': '${BUILDKITE_BRANCH}'},
|
||||
},
|
||||
}
|
||||
for e in os.environ:
|
||||
if e.startswith('ph_'):
|
||||
run_build_step['build']['env'][e] = os.getenv(e)
|
||||
steps.append(create_branch_step)
|
||||
steps.append(run_build_step)
|
||||
print(yaml.dump({'steps': steps}))
|
|
@ -30,6 +30,7 @@ from git import Repo, BadName, GitCommandError
|
|||
# FIXME: maybe move to config file
|
||||
"""URL of upstream LLVM repository."""
|
||||
LLVM_GITHUB_URL = 'ssh://git@github.com/llvm/llvm-project'
|
||||
FORK_REMOTE_URL = 'ssh://git@github.com/llvm-premerge-tests/llvm-project'
|
||||
|
||||
"""How far back the script searches in the git history to find Revisions that
|
||||
have already landed. """
|
||||
|
@ -49,10 +50,10 @@ class ApplyPatch:
|
|||
Once this class has applied all dependencies, it will apply the diff itself.
|
||||
|
||||
This script must be called from the root folder of a local checkout of
|
||||
https://github.com/llvm/llvm-project
|
||||
https://github.com/llvm/llvm-project or given a path to clone into.
|
||||
"""
|
||||
|
||||
def __init__(self, diff_id: int, comment_file_path: str, token: str, url: str, git_hash: str,
|
||||
def __init__(self, path: str, diff_id: int, comment_file_path: str, token: str, url: str, git_hash: str,
|
||||
push_branch: bool = False):
|
||||
self.comment_file_path = comment_file_path
|
||||
self.push_branch = push_branch # type: bool
|
||||
|
@ -65,7 +66,16 @@ class ApplyPatch:
|
|||
self.phab = self._create_phab()
|
||||
self.base_revision = git_hash # type: Optional[str]
|
||||
self.msg = [] # type: List[str]
|
||||
self.repo = Repo(os.getcwd()) # type: Repo
|
||||
|
||||
if not os.path.isdir(path):
|
||||
print(f'{path} does not exist, cloning repository')
|
||||
# TODO: progress of clonning
|
||||
self.repo = Repo.clone_from(FORK_REMOTE_URL, path)
|
||||
else:
|
||||
print('repository exist, will reuse')
|
||||
self.repo = Repo(path) # type: Repo
|
||||
os.chdir(path)
|
||||
print('working dir', os.getcwd())
|
||||
|
||||
@property
|
||||
def branch_name(self):
|
||||
|
@ -308,6 +318,7 @@ if __name__ == "__main__":
|
|||
parser = argparse.ArgumentParser(description='Apply Phabricator patch to working directory.')
|
||||
parser.add_argument('diff_id', type=int)
|
||||
# TODO: instead of --comment-file use stdout / stderr.
|
||||
parser.add_argument('--path', type=str, help='repository path', default=os.getcwd())
|
||||
parser.add_argument('--comment-file', type=str, dest='comment_file_path', default=None)
|
||||
parser.add_argument('--token', type=str, default=None, help='Conduit API token')
|
||||
parser.add_argument('--url', type=str, default=None, help='Phabricator URL')
|
||||
|
@ -316,5 +327,5 @@ if __name__ == "__main__":
|
|||
parser.add_argument('--push-branch', action='store_true', dest='push_branch',
|
||||
help='choose if branch shall be pushed to origin')
|
||||
args = parser.parse_args()
|
||||
patcher = ApplyPatch(args.diff_id, args.comment_file_path, args.token, args.url, args.commit, args.push_branch)
|
||||
patcher = ApplyPatch(args.path, args.diff_id, args.comment_file_path, args.token, args.url, args.commit, args.push_branch)
|
||||
patcher.run()
|
||||
|
|
Loading…
Reference in a new issue