6e624c30f9
The current setup is configuring the "affected projects" as well as their dependencies, and run `ninja all` followed by `ninja check-all`. This is quite a pessimization for leaf project which don't need to build and run the tests for their dependencies. For example a patch affecting only MLIR shouldn't need to build and run LLVM and clang tests. This patch changes this by running checks only for the affected project. For example a patch touching `mlir` is affecting `mlir` and `flang`. However `flang` depends on `clang`. So the list of projects to configure is `mlir;flang;clang;llvm;`, but we want to test only mlir and flang ; we'll run only `ninja check-mlir check-flang`. In practice in this example running `ninja all` builds 5658 targets and `ninja check-all` after that adds 716 more targets. On the other hands `ninja check-flang check-mlir` results in 3997 targets total. Concretely the contract with premerge_checks.py is changed so that the expected argument for the --projects flag is only the list of affected project, dependencies are automatically added.
99 lines
3.9 KiB
Python
Executable file
99 lines
3.9 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
# Copyright 2020 Google LLC
|
|
#
|
|
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# https://llvm.org/LICENSE.txt
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Script runs in checked out llvm-project directory.
|
|
|
|
import logging
|
|
import os
|
|
|
|
from buildkite_utils import annotate, feedback_url, set_metadata
|
|
from choose_projects import ChooseProjects
|
|
import git
|
|
from steps import generic_linux, generic_windows, from_shell_output, checkout_scripts, bazel, extend_steps_env
|
|
import yaml
|
|
|
|
steps_generators = [
|
|
'${BUILDKITE_BUILD_CHECKOUT_PATH}/libcxx/utils/ci/buildkite-pipeline-premerge.sh',
|
|
]
|
|
|
|
if __name__ == '__main__':
|
|
scripts_refspec = os.getenv("ph_scripts_refspec", "main")
|
|
diff_id = os.getenv("ph_buildable_diff", "")
|
|
no_cache = os.getenv('ph_no_cache') is not None
|
|
projects = os.getenv('ph_projects', 'detect')
|
|
log_level = os.getenv('ph_log_level', 'INFO')
|
|
logging.basicConfig(level=log_level, format='%(levelname)-7s %(message)s')
|
|
|
|
phid = os.getenv('ph_target_phid')
|
|
url = f"https://reviews.llvm.org/D{os.getenv('ph_buildable_revision')}?id={diff_id}"
|
|
annotate(f"Build for [D{os.getenv('ph_buildable_revision')}#{diff_id}]({url}). "
|
|
f"[Harbormaster build](https://reviews.llvm.org/harbormaster/build/{os.getenv('ph_build_id')}).\n"
|
|
f"If there is a build infrastructure issue, please [create a bug]({feedback_url()}).")
|
|
set_metadata('ph_buildable_diff', os.getenv("ph_buildable_diff"))
|
|
set_metadata('ph_buildable_revision', os.getenv('ph_buildable_revision'))
|
|
set_metadata('ph_build_id', os.getenv("ph_build_id"))
|
|
|
|
env = {}
|
|
for e in os.environ:
|
|
if e.startswith('ph_'):
|
|
env[e] = os.getenv(e)
|
|
repo = git.Repo('.')
|
|
steps = []
|
|
# List all affected projects.
|
|
patch = repo.git.diff("HEAD~1")
|
|
cp = ChooseProjects('.')
|
|
|
|
linux_projects, dependencies = cp.choose_projects(patch = patch, current_os = "linux")
|
|
logging.info(f'linux_projects: {linux_projects} (dependencies: {dependencies}')
|
|
if len(linux_projects) > 0:
|
|
steps.extend(generic_linux(';'.join(sorted(linux_projects)), True))
|
|
|
|
windows_projects, dependencies = cp.choose_projects(patch = patch, current_os = "windows")
|
|
logging.info(f'windows_projects: {windows_projects} (dependencies: {dependencies}')
|
|
if len(windows_projects) > 0:
|
|
steps.extend(generic_windows(';'.join(sorted(windows_projects))))
|
|
|
|
# Add custom checks.
|
|
if os.getenv('ph_skip_generated') is None:
|
|
e = os.environ.copy()
|
|
# BUILDKITE_COMMIT might be an alias, e.g. "HEAD". Resolve it to make the build hermetic.
|
|
e["BUILDKITE_COMMIT"] = repo.head.commit.hexsha
|
|
for gen in steps_generators:
|
|
steps.extend(from_shell_output(gen, env=e))
|
|
modified_files = cp.get_changed_files(patch)
|
|
steps.extend(bazel(modified_files))
|
|
|
|
if phid is None:
|
|
logging.warning('ph_target_phid is not specified. Skipping "Report" step')
|
|
else:
|
|
steps.append({
|
|
'wait': '~',
|
|
'continue_on_failure': True,
|
|
})
|
|
|
|
report_step = {
|
|
'label': ':phabricator: update build status on Phabricator',
|
|
'commands': [
|
|
*checkout_scripts('linux', scripts_refspec),
|
|
'$${SRC}/scripts/summary.py',
|
|
],
|
|
'artifact_paths': ['artifacts/**/*'],
|
|
'agents': {'queue': 'service'},
|
|
'timeout_in_minutes': 10,
|
|
}
|
|
steps.append(report_step)
|
|
|
|
extend_steps_env(steps, env)
|
|
print(yaml.dump({'steps': steps}))
|