Additional logging for project selection

+ option to skip win / linux build from diff checks

scripts/buildkite/build_branch_pipeline.py

@@ -87,11 +87,16 @@ if __name__ == '__main__':
         'agents': {'queue': f'{queue_prefix}windows'},
         'timeout_in_minutes': 120,
     }
-    steps.append(linux_buld_step)
-    steps.append(windows_buld_step)
+    deps = []
+    if os.getenv('ph_skip_linux') is None:
+        steps.append(linux_buld_step)
+        deps.append(linux_buld_step['key'])
+    if os.getenv('ph_skip_windows') is None:
+        steps.append(windows_buld_step)
+        deps.append(windows_buld_step['key'])
     report_step = {
         'label': ':spiral_note_pad: report',
-        'depends_on': [linux_buld_step['key'], windows_buld_step['key']],
+        'depends_on': deps,
         'commands': [
             'mkdir -p artifacts',
             'buildkite-agent artifact download "*_result.json" .',

scripts/choose_projects.py

@@ -62,7 +62,8 @@ class ChooseProjects:
             for used in used_list:
                 self.usages.setdefault(used, []).append(user)
         self.all_projects = config['allprojects']
-        self.excluded_projects = set(config['excludedProjects'][self.operating_system])
+        excluded = config['excludedProjects'][self.operating_system]
+        self.excluded_projects = set(excluded if excluded is not None else [])
 
     @staticmethod
     def _detect_os() -> str:
@@ -86,10 +87,19 @@ class ChooseProjects:
             logging.warning('There were changes that could not be mapped to a project.'
                             'Building all projects instead!')
             return self.FALLBACK_PROJECTS
+        return self.extend_projects(changed_projects)
 
-        affected_projects = self.get_affected_projects(changed_projects)
+    def extend_projects(self, projects: Set[str]) -> List[str]:
+        logging.info(f'projects: {projects}')
+        affected_projects = self.get_affected_projects(projects)
+        logging.info(f'with affected projects: {affected_projects}')
         affected_projects = self.add_dependencies(affected_projects)
-        affected_projects = affected_projects - self.excluded_projects
+        logging.info(f'with dependencies: {affected_projects}')
+        to_exclude = affected_projects.intersection(self.excluded_projects)
+        if len(to_exclude) != 0:
+            affected_projects = affected_projects - to_exclude
+            logging.warning(f'{to_exclude} projects are excluded')
+            logging.info(f'without excluded: {affected_projects}')
         return sorted(affected_projects)
 
     def run(self):
@@ -147,9 +157,7 @@ class ChooseProjects:
                     changes.update(self.usages[project])
             affected_projects.update(changes)
 
-        logging.info('Projects affected by this patch:')
-        logging.info('  ' + '\n  '.join(sorted(affected_projects)))
-
+        logging.info(f'added {affected_projects - changed_projects} projects as they are affected')
         return affected_projects
 
     def add_dependencies(self, projects: Set[str]) -> Set[str]:
@@ -170,8 +178,7 @@ class ChooseProjects:
 
     def get_all_enabled_projects(self) -> List[str]:
         """Get list of all not-excluded projects for current platform."""
-        result = set(self.all_projects) - self.excluded_projects
-        return sorted(list(result))
+        return self.extend_projects(set(self.all_projects))
 
 
 if __name__ == "__main__":

scripts/run_cmake.py

@@ -90,8 +90,10 @@ def _select_projects(config: Configuration, projects: str, repo_path: str) -> st
         cp = ChooseProjects(repo_path)
         repo = Repo('.')
         patch = repo.git.diff("HEAD~1")
+        logging.debug(f'diff {patch}')
         enabled_projects = ';'.join(cp.choose_projects(patch))
         if enabled_projects is None or len(enabled_projects) == 0:
+            logging.warning('Cannot detect affected projects. Enable all projects')
             enabled_projects = cp.get_all_enabled_projects()
         return enabled_projects
     return projects
@@ -164,7 +166,7 @@ def run(projects: str, repo_path: str, config_file_path: str = None, *, dry_run:
     print('Running cmake with these arguments:\n{}'.format(cmd), flush=True)
     if dry_run:
         print('Dry run, not invoking CMake!')
-        return 0, build_dir, []
+        return 0, build_dir, [], []
     result = subprocess.call(cmd, env=env, shell=True, cwd=build_dir)
     commands.append('cmake ' + ' '.join(_create_args(config, llvm_enable_projects, False)))
     commands.append('# ^note that compiler cache arguments are omitted')
@@ -203,6 +205,8 @@ if __name__ == '__main__':
     parser.add_argument('projects', type=str, nargs='?', default='default')
     parser.add_argument('repo_path', type=str, nargs='?', default=os.getcwd())
     parser.add_argument('--dryrun', action='store_true')
+    parser.add_argument('--log-level', type=str, default='WARNING')
     args = parser.parse_args()
-    result, _, _ = run(args.projects, args.repo_path, dry_run=args.dryrun)
+    logging.basicConfig(level=args.log_level, format='%(levelname)-7s %(message)s')
+    result, _, _, _ = run(args.projects, args.repo_path, dry_run=args.dryrun)
     sys.exit(result)