From a638dd53429fb78925768b7d9b180c4dca08f062 Mon Sep 17 00:00:00 2001 From: ChristianKuehnel Date: Mon, 23 Mar 2020 09:03:24 +0100 Subject: [PATCH] migrate scripts to python (#148) * run_cmake.py working locally on Linux * added os detection of choose_projects * cmake works on windows * run_ninja works on windows * fixed newline characters --- scripts/choose_projects.py | 73 ++++++++++------- scripts/llvm-dependencies.yaml | 20 +++-- scripts/run_cmake.py | 145 +++++++++++++++++++++++++++++++++ scripts/run_cmake_config.yaml | 36 ++++++++ scripts/run_ninja.py | 38 +++++++++ 5 files changed, 274 insertions(+), 38 deletions(-) create mode 100755 scripts/run_cmake.py create mode 100644 scripts/run_cmake_config.yaml create mode 100755 scripts/run_ninja.py diff --git a/scripts/choose_projects.py b/scripts/choose_projects.py index a880325..85182ef 100755 --- a/scripts/choose_projects.py +++ b/scripts/choose_projects.py @@ -23,6 +23,7 @@ project dependency graph it will get the transitively affected projects. import argparse import logging import os +import platform import sys from typing import Dict, List, Set, Tuple @@ -30,25 +31,28 @@ from unidiff import PatchSet import yaml # TODO: We could also try to avoid running tests for llvm for projects that -# only need various cmake scripts and don't actually depend on llvm (e.g. -# libcxx does not need to run llvm tests, but may still need to include llvm). +# only need various cmake scripts and don't actually depend on llvm (e.g. +# libcxx does not need to run llvm tests, but may still need to include llvm). + class ChooseProjects: # file where dependencies are defined SCRIPT_DIR = os.path.dirname(__file__) DEPENDENCIES_FILE = os.path.join(SCRIPT_DIR, 'llvm-dependencies.yaml') + # projects used if anything goes wrong + FALLBACK_PROJECTS = ['all'] def __init__(self, llvm_dir: str): self.llvm_dir = llvm_dir - self.defaultProjects = dict() # type: Dict[str, Dict[str, str]] - self.dependencies = dict() # type: Dict[str,List[str]] + self.defaultProjects = dict() # type: Dict[str, Dict[str, str]] + self.dependencies = dict() # type: Dict[str,List[str]] self.usages = dict() # type: Dict[str,List[str]] - self.all_projects = [] # type: List[str] - self.excluded_projects = set() # type: Set[str] + self.all_projects = [] # type: List[str] + self.excluded_projects = set() # type: Set[str] + self.operating_system = self._detect_os() # type: str self._load_config() - def _load_config(self): logging.info('loading project config from {}'.format(self.DEPENDENCIES_FILE)) with open(self.DEPENDENCIES_FILE) as dependencies_file: @@ -56,30 +60,38 @@ class ChooseProjects: self.dependencies = config['dependencies'] for user, used_list in self.dependencies.items(): for used in used_list: - self.usages.setdefault(used,[]).append(user) + self.usages.setdefault(used, []).append(user) self.all_projects = config['allprojects'] - self.excluded_projects = set(config['excludedProjects']) + self.excluded_projects = set(config['excludedProjects'][self.operating_system]) - def run(self): - llvm_dir = os.path.abspath(os.path.expanduser(args.llvmdir)) + @staticmethod + def _detect_os() -> str: + """Detect the current operating system.""" + if platform.system() == 'Windows': + return 'windows' + return 'linux' + + def choose_projects(self, patch: str = None) -> List[str]: + llvm_dir = os.path.abspath(os.path.expanduser(self.llvm_dir)) logging.info('Scanning LLVM in {}'.format(llvm_dir)) if not self.match_projects_dirs(): - sys.exit(1) - changed_files = self.get_changed_files() + return self.FALLBACK_PROJECTS + changed_files = self.get_changed_files(patch) changed_projects, unmapped_changes = self.get_changed_projects(changed_files) if unmapped_changes: logging.warning('There were changes that could not be mapped to a project.' 'Building all projects instead!') - print('all') - return 0 + return self.FALLBACK_PROJECTS affected_projects = self.get_affected_projects(changed_projects) affected_projects = self.add_dependencies(affected_projects) affected_projects = affected_projects - self.excluded_projects - print(';'.join(sorted(affected_projects))) - return 0 + return sorted(affected_projects) + def run(self): + print(';'.join(self.choose_projects())) + return 0 def match_projects_dirs(self) -> bool: """Make sure that all projects are folders in the LLVM dir. @@ -91,23 +103,25 @@ class ChooseProjects: logging.error('Project no found in LLVM root folder: {}'.format(project)) return False return True - @staticmethod - def get_changed_files() -> Set[str]: + def get_changed_files(patch_str: str = None) -> Set[str]: """get list of changed files from the patch from STDIN.""" - patch = PatchSet(sys.stdin) + if patch_str is None: + patch_str = PatchSet(sys.stdin) + patch = PatchSet(patch_str) + changed_files = set({f.path for f in patch.modified_files + patch.added_files + patch.removed_files}) logging.info('Files modified by this patch:\n ' + '\n '.join(sorted(changed_files))) return changed_files - def get_changed_projects(self, changed_files: Set[str]) -> Tuple[Set[str],bool]: + def get_changed_projects(self, changed_files: Set[str]) -> Tuple[Set[str], bool]: """Get list of projects affected by the change.""" changed_projects = set() unmapped_changes = False for changed_file in changed_files: - project = changed_file.split('/',maxsplit=1) + project = changed_file.split('/', maxsplit=1) if project is None or project[0] not in self.all_projects: unmapped_changes = True logging.warning('Could not map file to project: {}'.format(changed_file)) @@ -117,10 +131,10 @@ class ChooseProjects: logging.info('Projects directly modified by this patch:\n ' + '\n '.join(sorted(changed_projects))) return changed_projects, unmapped_changes - - def get_affected_projects(self, changed_projects:Set[str]) -> Set[str]: - """Compute transitive closure of affected projects based on the dependencies between the projects.""" - affected_projects=set(changed_projects) + def get_affected_projects(self, changed_projects: Set[str]) -> Set[str]: + """Compute transitive closure of affected projects based on the + dependencies between the projects.""" + affected_projects = set(changed_projects) last_len = -1 while len(affected_projects) != last_len: last_len = len(affected_projects) @@ -135,7 +149,6 @@ class ChooseProjects: return affected_projects - def add_dependencies(self, projects: Set[str]) -> Set[str]: """Return projects and their dependencies. @@ -152,10 +165,12 @@ class ChooseProjects: result.update(changes) return result + if __name__ == "__main__": logging.basicConfig(filename='choose_projects.log', level=logging.INFO) - parser = argparse.ArgumentParser(description='Compute the projects affected by a change.') + parser = argparse.ArgumentParser( + description='Compute the projects affected by a change.') parser.add_argument('llvmdir', default='.') args = parser.parse_args() chooser = ChooseProjects(args.llvmdir) - sys.exit(chooser.run()) \ No newline at end of file + sys.exit(chooser.run()) diff --git a/scripts/llvm-dependencies.yaml b/scripts/llvm-dependencies.yaml index 55e4dcb..1c5bf89 100644 --- a/scripts/llvm-dependencies.yaml +++ b/scripts/llvm-dependencies.yaml @@ -42,7 +42,6 @@ dependencies: # List of all projects in the LLVM monorepository. This list is taken from # llvm/CMakeLists.txt in "set(LLVM_ALL_PROJECTS ..." - allprojects: - clang - clang-tools-extra @@ -64,11 +63,14 @@ allprojects: # projects excluded from automatic configuration as they could not be built excludedProjects: -# These projects are not working with Visual Studio Compiler on Windows - - lldb - - llgo - - libunwind - - libcxxabi - - openmp # blacklisting as kuhnel has trouble with the Perl installation - - debuginfo-tests # test failing - - polly # test failing \ No newline at end of file + # These projects are not working with Visual Studio Compiler on Windows + windows: + - lldb + - llgo + - libunwind + - libcxxabi + - openmp # blacklisting as kuhnel has trouble with the Perl installation + - debuginfo-tests # test failing + - polly # test failing + # no projects are excluded on Linux + linux: [] diff --git a/scripts/run_cmake.py b/scripts/run_cmake.py new file mode 100755 index 0000000..a86d9f7 --- /dev/null +++ b/scripts/run_cmake.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# Copyright 2020 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +from enum import Enum +from git import Repo +import os +import platform +import shutil +import subprocess +from typing import List, Dict +import yaml +from choose_projects import ChooseProjects + + +class OperatingSystem(Enum): + Linux = 'linux' + Windows = 'windows' + + +class Configuration: + + def __init__(self, config_file_path: str): + with open(config_file_path) as config_file: + config = yaml.load(config_file, Loader=yaml.SafeLoader) + self._environment = config['environment'] # type: Dict[OperatingSystem, Dict[str, str]] + self.general_cmake_arguments = config['arguments']['general'] # type: List[str] + self._specific_cmake_arguments = config['arguments'] # type: Dict[OperatingSystem, List[str]] + self._default_projects = config['default_projects'] # type: Dict[OperatingSystem, str] + self.operating_system = self._detect_os() # type: OperatingSystem + + @property + def environment(self) -> Dict[str, str]: + return self._environment[self.operating_system.value] + + @property + def specific_cmake_arguments(self) -> List[str]: + return self._specific_cmake_arguments[self.operating_system.value] + + @property + def default_projects(self) -> str: + return self._default_projects[self.operating_system.value] + + @staticmethod + def _detect_os() -> OperatingSystem: + """Detect the current operating system.""" + if platform.system() == 'Windows': + return OperatingSystem.Windows + return OperatingSystem.Linux + + +def _select_projects(config: Configuration, projects: str, repo_path: str) -> str: + """select which projects to build. + + if projects == "default", a default configuraiton will be used. + + if project == "detect", ChooseProjects is used to magically detect the projects + based on the files modified in HEAD + """ + if projects == "default" or projects is None or len(projects) == 0: + return config.default_projects + if projects == "detect": + cp = ChooseProjects(repo_path) + repo = Repo('.') + patch = repo.git.diff("HEAD~1") + enabled_projects = ';'.join(cp.choose_projects(patch)) + if enabled_projects is None or len(enabled_projects) == 0: + enabled_projects = 'all' + return enabled_projects + return projects + + +def _create_env(config: Configuration) -> Dict[str, str]: + """Generate the environment variables for cmake.""" + env = os.environ.copy() + env.update(config.environment) + return env + + +def _create_args(config: Configuration, llvm_enable_projects: str) -> List[str]: + """Generate the command line arguments for cmake.""" + arguments = [ + os.path.join('..', 'llvm'), + '-D LLVM_ENABLE_PROJECTS="{}"'.format(llvm_enable_projects), + ] + arguments.extend(config.general_cmake_arguments) + arguments.extend(config.specific_cmake_arguments) + + # enable ccache if the path is set in the environment + if 'CCACHE_PATH' in os.environ: + arguments.extend([ + '-D LLVM_CCACHE_BUILD=ON', + '-D LLVM_CCACHE_DIR={}'.format(os.environ['CCACHE_PATH']), + '-D LLVM_CCACHE_MAXSIZE=20G', + ]) + return arguments + + +def run_cmake(projects: str, repo_path: str, config_file_path: str = None): + """Use cmake to configure the project. + + This version works on all operating systems. + """ + if config_file_path is None: + script_dir = os.path.dirname(__file__) + config_file_path = os.path.join(script_dir, 'run_cmake_config.yaml') + config = Configuration(config_file_path) + + build_dir = os.path.abspath(os.path.join(repo_path, 'build')) + if os.path.exists(build_dir): + shutil.rmtree(build_dir) + os.makedirs(build_dir) + + env = _create_env(config) + llvm_enable_projects = _select_projects(config, projects, repo_path) + arguments = _create_args(config, llvm_enable_projects) + cmd = 'cmake ' + ' '.join(arguments) + + # On Windows: configure Visutal Studio before running cmake + if config.operating_system == OperatingSystem.Windows: + # FIXME: move this path to a config file + cmd = r'"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64 && ' + cmd + + subprocess.check_call(cmd, env=env, shell=True, cwd=build_dir) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Run CMake for LLVM.') + parser.add_argument('projects', type=str, nargs='?', default='default') + parser.add_argument('repo_path', type=str, nargs='?', default=os.getcwd()) + args = parser.parse_args() + run_cmake(args.projects, args.repo_path) + diff --git a/scripts/run_cmake_config.yaml b/scripts/run_cmake_config.yaml new file mode 100644 index 0000000..b96c65f --- /dev/null +++ b/scripts/run_cmake_config.yaml @@ -0,0 +1,36 @@ +# This file is used to configure the environment variables and define +# the command line arguments for cmake + +# environment variables that are set per operating system +environment: + linux: + CC: 'clang' + CXX: 'clang++' + LD: 'LLD' + windows: + 'CC': 'cl' + 'CXX': 'cl' + 'LD': 'link' + +# command line arguments for cmake +arguments: + # command line arguments for all OS + general: + # LLVM path is set in script + # LLVM_ENABLE_PROJECTS is set in script + - '-G Ninja' + - '-D CMAKE_BUILD_TYPE=Release' + - '-D LLVM_ENABLE_ASSERTIONS=ON' + - '-D LLVM_LIT_ARGS="-v --xunit-xml-output test-results.xml"' + linux: + # CCACHE is enabled in script iff environment variable `CCACHE_PATH` is set + - '-D LLVM_ENABLE_LLD=ON' + - '-DCMAKE_CXX_FLAGS=-gmlt' + windows: + - '-D LLVM_ENABLE_DIA_SDK=OFF' + +# if the automatic project detection fails or is not used, these projects are +# enabled +default_projects: + windows: 'clang;clang-tools-extra;libcxx;libc;lld;mlir;libcxxabi' + linux: 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir' diff --git a/scripts/run_ninja.py b/scripts/run_ninja.py new file mode 100755 index 0000000..16b125f --- /dev/null +++ b/scripts/run_ninja.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# Copyright 2020 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import platform +import subprocess + +def run_ninja(target: str, repo_path: str): + build_dir = os.path.join(repo_path, 'build') + cmd = 'ninja {}'.format(target) + + # On Windows: configure Visutal Studio before running ninja + if platform.system() == 'Windows': + # FIXME: move this path to a config file + cmd = r'"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64 && ' + cmd + + subprocess.check_call(cmd, shell=True, cwd=build_dir) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Run ninja for LLVM.') + parser.add_argument('target') + parser.add_argument('repo_path', type=str, nargs='?', default=os.getcwd()) + args = parser.parse_args() + run_ninja(args.target, args.repo_path)