migrate scripts to python (#148)

* run_cmake.py working locally on Linux

* added os detection of choose_projects

* cmake works on windows

* run_ninja works on windows

* fixed newline characters

scripts/choose_projects.py

@@ -23,6 +23,7 @@ project dependency graph it will get the transitively affected projects.
 import argparse
 import logging
 import os
+import platform
 import sys
 from typing import Dict, List, Set, Tuple
 
@@ -33,11 +34,14 @@ import yaml
 #   only need various cmake scripts and don't actually depend on llvm (e.g.
 #   libcxx does not need to run llvm tests, but may still need to include llvm).
 
+
 class ChooseProjects:
     
     # file where dependencies are defined
     SCRIPT_DIR = os.path.dirname(__file__)
     DEPENDENCIES_FILE = os.path.join(SCRIPT_DIR, 'llvm-dependencies.yaml')
+    # projects used if anything goes wrong
+    FALLBACK_PROJECTS = ['all']
 
     def __init__(self, llvm_dir: str):
         self.llvm_dir = llvm_dir
@@ -46,9 +50,9 @@ class ChooseProjects:
         self.usages = dict()   # type: Dict[str,List[str]]
         self.all_projects = []  # type: List[str]
         self.excluded_projects = set()  # type: Set[str]
+        self.operating_system = self._detect_os()  # type: str
         self._load_config()
 
-
     def _load_config(self):
         logging.info('loading project config from {}'.format(self.DEPENDENCIES_FILE))
         with open(self.DEPENDENCIES_FILE) as dependencies_file:
@@ -56,30 +60,38 @@ class ChooseProjects:
         self.dependencies = config['dependencies']
         for user, used_list in self.dependencies.items():
             for used in used_list:
-                self.usages.setdefault(used,[]).append(user)
+                self.usages.setdefault(used, []).append(user)
         self.all_projects = config['allprojects']
-        self.excluded_projects = set(config['excludedProjects'])
+        self.excluded_projects = set(config['excludedProjects'][self.operating_system])
 
-    def run(self):
-        llvm_dir = os.path.abspath(os.path.expanduser(args.llvmdir))
+    @staticmethod
+    def _detect_os() -> str:
+        """Detect the current operating system."""
+        if platform.system() == 'Windows':
+            return 'windows'
+        return 'linux'
+
+    def choose_projects(self, patch: str = None) -> List[str]:
+        llvm_dir = os.path.abspath(os.path.expanduser(self.llvm_dir))
         logging.info('Scanning LLVM in {}'.format(llvm_dir))
         if not self.match_projects_dirs():
-            sys.exit(1)
-        changed_files = self.get_changed_files()
+            return self.FALLBACK_PROJECTS
+        changed_files = self.get_changed_files(patch)
         changed_projects, unmapped_changes = self.get_changed_projects(changed_files)
 
         if unmapped_changes:
             logging.warning('There were changes that could not be mapped to a project.'
                             'Building all projects instead!')
-            print('all')
-            return 0
+            return self.FALLBACK_PROJECTS
 
         affected_projects = self.get_affected_projects(changed_projects)
         affected_projects = self.add_dependencies(affected_projects)
         affected_projects = affected_projects - self.excluded_projects
-        print(';'.join(sorted(affected_projects)))
-        return 0
+        return sorted(affected_projects)
 
+    def run(self):
+        print(';'.join(self.choose_projects()))
+        return 0
 
     def match_projects_dirs(self) -> bool:
         """Make sure that all projects are folders in the LLVM dir.
@@ -92,22 +104,24 @@ class ChooseProjects:
                 return False
         return True
 
-
     @staticmethod
-    def get_changed_files() -> Set[str]:
+    def get_changed_files(patch_str: str = None) -> Set[str]:
         """get list of changed files from the patch from STDIN."""
-        patch = PatchSet(sys.stdin)
+        if patch_str is None:
+            patch_str = PatchSet(sys.stdin)
+        patch = PatchSet(patch_str)
+
         changed_files = set({f.path for f in patch.modified_files + patch.added_files + patch.removed_files})
 
         logging.info('Files modified by this patch:\n  ' + '\n  '.join(sorted(changed_files)))
         return changed_files
 
-    def get_changed_projects(self, changed_files: Set[str]) -> Tuple[Set[str],bool]:
+    def get_changed_projects(self, changed_files: Set[str]) -> Tuple[Set[str], bool]:
         """Get list of projects affected by the change."""
         changed_projects = set()
         unmapped_changes = False
         for changed_file in changed_files:
-            project = changed_file.split('/',maxsplit=1)
+            project = changed_file.split('/', maxsplit=1)
             if project is None or project[0] not in self.all_projects:
                 unmapped_changes = True
                 logging.warning('Could not map file to project: {}'.format(changed_file))
@@ -117,10 +131,10 @@ class ChooseProjects:
         logging.info('Projects directly modified by this patch:\n  ' + '\n  '.join(sorted(changed_projects)))
         return changed_projects, unmapped_changes
 
-
-    def get_affected_projects(self, changed_projects:Set[str]) -> Set[str]:
-        """Compute transitive closure of affected projects based on the dependencies between the projects."""
-        affected_projects=set(changed_projects)
+    def get_affected_projects(self, changed_projects: Set[str]) -> Set[str]:
+        """Compute transitive closure of affected projects based on the
+        dependencies between the projects."""
+        affected_projects = set(changed_projects)
         last_len = -1
         while len(affected_projects) != last_len:
             last_len = len(affected_projects)
@@ -135,7 +149,6 @@ class ChooseProjects:
 
         return affected_projects
 
-
     def add_dependencies(self, projects: Set[str]) -> Set[str]:
         """Return projects and their dependencies.
 
@@ -152,9 +165,11 @@ class ChooseProjects:
             result.update(changes)
         return result
 
+
 if __name__ == "__main__":
     logging.basicConfig(filename='choose_projects.log', level=logging.INFO)
-    parser = argparse.ArgumentParser(description='Compute the projects affected by a change.')
+    parser = argparse.ArgumentParser(
+        description='Compute the projects affected by a change.')
     parser.add_argument('llvmdir', default='.')
     args = parser.parse_args()
     chooser = ChooseProjects(args.llvmdir)

scripts/llvm-dependencies.yaml

@@ -42,7 +42,6 @@ dependencies:
 
 # List of all projects in the LLVM monorepository. This list is taken from
 # llvm/CMakeLists.txt in "set(LLVM_ALL_PROJECTS ..."
-
 allprojects:
   - clang
   - clang-tools-extra
@@ -64,7 +63,8 @@ allprojects:
 
 # projects excluded from automatic configuration as they could not be built
 excludedProjects:
-# These projects are not working with Visual Studio Compiler on Windows
+  # These projects are not working with Visual Studio Compiler on Windows
+  windows:
     - lldb
     - llgo
     - libunwind
@@ -72,3 +72,5 @@ excludedProjects:
     - openmp # blacklisting as kuhnel has trouble with the Perl installation
     - debuginfo-tests # test failing
     - polly # test failing
+  # no projects are excluded on Linux
+  linux: []

scripts/run_cmake.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+# Copyright 2020 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+from enum import Enum
+from git import Repo
+import os
+import platform
+import shutil
+import subprocess
+from typing import List, Dict
+import yaml
+from choose_projects import ChooseProjects
+
+
+class OperatingSystem(Enum):
+    Linux = 'linux'
+    Windows = 'windows'
+
+
+class Configuration:
+
+    def __init__(self, config_file_path: str):
+        with open(config_file_path) as config_file:
+            config = yaml.load(config_file, Loader=yaml.SafeLoader)
+        self._environment = config['environment']  # type: Dict[OperatingSystem, Dict[str, str]]
+        self.general_cmake_arguments = config['arguments']['general']  # type: List[str]
+        self._specific_cmake_arguments = config['arguments']  # type: Dict[OperatingSystem, List[str]]
+        self._default_projects = config['default_projects']  # type: Dict[OperatingSystem, str]
+        self.operating_system = self._detect_os()  # type: OperatingSystem
+
+    @property
+    def environment(self) -> Dict[str, str]:
+        return self._environment[self.operating_system.value]
+
+    @property
+    def specific_cmake_arguments(self) -> List[str]:
+        return self._specific_cmake_arguments[self.operating_system.value]
+
+    @property
+    def default_projects(self) -> str:
+        return self._default_projects[self.operating_system.value]
+
+    @staticmethod
+    def _detect_os() -> OperatingSystem:
+        """Detect the current operating system."""
+        if platform.system() == 'Windows':
+            return OperatingSystem.Windows
+        return OperatingSystem.Linux
+
+
+def _select_projects(config: Configuration, projects: str, repo_path: str) -> str:
+    """select which projects to build.
+
+    if projects == "default", a default configuraiton will be used.
+
+    if project == "detect", ChooseProjects is used to magically detect the projects
+         based on the files modified in HEAD
+    """
+    if projects == "default" or projects is None or len(projects) == 0:
+        return config.default_projects
+    if projects == "detect":
+        cp = ChooseProjects(repo_path)
+        repo = Repo('.')
+        patch = repo.git.diff("HEAD~1")
+        enabled_projects = ';'.join(cp.choose_projects(patch))
+        if enabled_projects is None or len(enabled_projects) == 0:
+            enabled_projects = 'all'
+        return enabled_projects
+    return projects
+
+
+def _create_env(config: Configuration) -> Dict[str, str]:
+    """Generate the environment variables for cmake."""
+    env = os.environ.copy()
+    env.update(config.environment)
+    return env
+
+
+def _create_args(config: Configuration, llvm_enable_projects: str) -> List[str]:
+    """Generate the command line arguments for cmake."""
+    arguments = [
+        os.path.join('..', 'llvm'),
+        '-D LLVM_ENABLE_PROJECTS="{}"'.format(llvm_enable_projects),
+    ]
+    arguments.extend(config.general_cmake_arguments)
+    arguments.extend(config.specific_cmake_arguments)
+
+    # enable ccache if the path is set in the environment
+    if 'CCACHE_PATH' in os.environ:
+        arguments.extend([
+            '-D LLVM_CCACHE_BUILD=ON',
+            '-D LLVM_CCACHE_DIR={}'.format(os.environ['CCACHE_PATH']),
+            '-D LLVM_CCACHE_MAXSIZE=20G',
+        ])
+    return arguments
+
+
+def run_cmake(projects: str, repo_path: str, config_file_path: str = None):
+    """Use cmake to configure the project.
+
+    This version works on all operating systems.
+    """
+    if config_file_path is None:
+        script_dir = os.path.dirname(__file__)
+        config_file_path = os.path.join(script_dir, 'run_cmake_config.yaml')
+    config = Configuration(config_file_path)
+
+    build_dir = os.path.abspath(os.path.join(repo_path, 'build'))
+    if os.path.exists(build_dir):
+        shutil.rmtree(build_dir)
+    os.makedirs(build_dir)
+
+    env = _create_env(config)
+    llvm_enable_projects = _select_projects(config, projects, repo_path)
+    arguments = _create_args(config, llvm_enable_projects)
+    cmd = 'cmake ' + ' '.join(arguments)
+    
+    # On Windows: configure Visutal Studio before running cmake
+    if config.operating_system == OperatingSystem.Windows:
+        # FIXME: move this path to a config file
+        cmd = r'"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64 && ' + cmd
+    
+    subprocess.check_call(cmd, env=env, shell=True, cwd=build_dir)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Run CMake for LLVM.')
+    parser.add_argument('projects', type=str, nargs='?', default='default')
+    parser.add_argument('repo_path', type=str, nargs='?', default=os.getcwd())
+    args = parser.parse_args()
+    run_cmake(args.projects, args.repo_path)
+

scripts/run_cmake_config.yaml

@@ -0,0 +1,36 @@
+# This file is used to configure the environment variables and define
+# the command line arguments for cmake
+
+# environment variables that are set per operating system
+environment:
+  linux:
+    CC: 'clang'
+    CXX: 'clang++'
+    LD: 'LLD'
+  windows:
+    'CC': 'cl'
+    'CXX': 'cl'
+    'LD': 'link'
+
+# command line arguments for cmake
+arguments:
+  # command line arguments for all OS
+  general:
+    # LLVM path is set in script
+    # LLVM_ENABLE_PROJECTS is set in script
+    - '-G Ninja'
+    - '-D CMAKE_BUILD_TYPE=Release'
+    - '-D LLVM_ENABLE_ASSERTIONS=ON'
+    - '-D LLVM_LIT_ARGS="-v --xunit-xml-output test-results.xml"'
+  linux:
+    # CCACHE is enabled in script iff environment variable `CCACHE_PATH` is set
+    - '-D LLVM_ENABLE_LLD=ON'
+    - '-DCMAKE_CXX_FLAGS=-gmlt'
+  windows:
+    - '-D LLVM_ENABLE_DIA_SDK=OFF'
+
+# if the automatic project detection fails or is not used, these projects are
+# enabled
+default_projects:
+  windows: 'clang;clang-tools-extra;libcxx;libc;lld;mlir;libcxxabi'
+  linux:   'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir'

scripts/run_ninja.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# Copyright 2020 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import platform
+import subprocess
+
+def run_ninja(target: str, repo_path: str):   
+    build_dir = os.path.join(repo_path, 'build')
+    cmd = 'ninja {}'.format(target)
+    
+    # On Windows: configure Visutal Studio before running ninja
+    if platform.system() == 'Windows':
+        # FIXME: move this path to a config file
+        cmd = r'"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64 && ' + cmd
+
+    subprocess.check_call(cmd, shell=True, cwd=build_dir)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Run ninja for LLVM.')
+    parser.add_argument('target')
+    parser.add_argument('repo_path', type=str, nargs='?', default=os.getcwd())
+    args = parser.parse_args()
+    run_ninja(args.target, args.repo_path)