diff --git a/scripts/buildkite/build_branch_pipeline.py b/scripts/buildkite/build_branch_pipeline.py index 29298b6..4611814 100755 --- a/scripts/buildkite/build_branch_pipeline.py +++ b/scripts/buildkite/build_branch_pipeline.py @@ -87,11 +87,16 @@ if __name__ == '__main__': 'agents': {'queue': f'{queue_prefix}windows'}, 'timeout_in_minutes': 120, } - steps.append(linux_buld_step) - steps.append(windows_buld_step) + deps = [] + if os.getenv('ph_skip_linux') is None: + steps.append(linux_buld_step) + deps.append(linux_buld_step['key']) + if os.getenv('ph_skip_windows') is None: + steps.append(windows_buld_step) + deps.append(windows_buld_step['key']) report_step = { 'label': ':spiral_note_pad: report', - 'depends_on': [linux_buld_step['key'], windows_buld_step['key']], + 'depends_on': deps, 'commands': [ 'mkdir -p artifacts', 'buildkite-agent artifact download "*_result.json" .', diff --git a/scripts/choose_projects.py b/scripts/choose_projects.py index eb628cf..893c76e 100755 --- a/scripts/choose_projects.py +++ b/scripts/choose_projects.py @@ -62,7 +62,8 @@ class ChooseProjects: for used in used_list: self.usages.setdefault(used, []).append(user) self.all_projects = config['allprojects'] - self.excluded_projects = set(config['excludedProjects'][self.operating_system]) + excluded = config['excludedProjects'][self.operating_system] + self.excluded_projects = set(excluded if excluded is not None else []) @staticmethod def _detect_os() -> str: @@ -86,10 +87,19 @@ class ChooseProjects: logging.warning('There were changes that could not be mapped to a project.' 'Building all projects instead!') return self.FALLBACK_PROJECTS + return self.extend_projects(changed_projects) - affected_projects = self.get_affected_projects(changed_projects) + def extend_projects(self, projects: Set[str]) -> List[str]: + logging.info(f'projects: {projects}') + affected_projects = self.get_affected_projects(projects) + logging.info(f'with affected projects: {affected_projects}') affected_projects = self.add_dependencies(affected_projects) - affected_projects = affected_projects - self.excluded_projects + logging.info(f'with dependencies: {affected_projects}') + to_exclude = affected_projects.intersection(self.excluded_projects) + if len(to_exclude) != 0: + affected_projects = affected_projects - to_exclude + logging.warning(f'{to_exclude} projects are excluded') + logging.info(f'without excluded: {affected_projects}') return sorted(affected_projects) def run(self): @@ -147,9 +157,7 @@ class ChooseProjects: changes.update(self.usages[project]) affected_projects.update(changes) - logging.info('Projects affected by this patch:') - logging.info(' ' + '\n '.join(sorted(affected_projects))) - + logging.info(f'added {affected_projects - changed_projects} projects as they are affected') return affected_projects def add_dependencies(self, projects: Set[str]) -> Set[str]: @@ -170,8 +178,7 @@ class ChooseProjects: def get_all_enabled_projects(self) -> List[str]: """Get list of all not-excluded projects for current platform.""" - result = set(self.all_projects) - self.excluded_projects - return sorted(list(result)) + return self.extend_projects(set(self.all_projects)) if __name__ == "__main__": diff --git a/scripts/run_cmake.py b/scripts/run_cmake.py index 4cf2a5e..366ad0c 100755 --- a/scripts/run_cmake.py +++ b/scripts/run_cmake.py @@ -90,8 +90,10 @@ def _select_projects(config: Configuration, projects: str, repo_path: str) -> st cp = ChooseProjects(repo_path) repo = Repo('.') patch = repo.git.diff("HEAD~1") + logging.debug(f'diff {patch}') enabled_projects = ';'.join(cp.choose_projects(patch)) if enabled_projects is None or len(enabled_projects) == 0: + logging.warning('Cannot detect affected projects. Enable all projects') enabled_projects = cp.get_all_enabled_projects() return enabled_projects return projects @@ -164,7 +166,7 @@ def run(projects: str, repo_path: str, config_file_path: str = None, *, dry_run: print('Running cmake with these arguments:\n{}'.format(cmd), flush=True) if dry_run: print('Dry run, not invoking CMake!') - return 0, build_dir, [] + return 0, build_dir, [], [] result = subprocess.call(cmd, env=env, shell=True, cwd=build_dir) commands.append('cmake ' + ' '.join(_create_args(config, llvm_enable_projects, False))) commands.append('# ^note that compiler cache arguments are omitted') @@ -203,6 +205,8 @@ if __name__ == '__main__': parser.add_argument('projects', type=str, nargs='?', default='default') parser.add_argument('repo_path', type=str, nargs='?', default=os.getcwd()) parser.add_argument('--dryrun', action='store_true') + parser.add_argument('--log-level', type=str, default='WARNING') args = parser.parse_args() - result, _, _ = run(args.projects, args.repo_path, dry_run=args.dryrun) + logging.basicConfig(level=args.log_level, format='%(levelname)-7s %(message)s') + result, _, _, _ = run(args.projects, args.repo_path, dry_run=args.dryrun) sys.exit(result)