get libcxx steps from sh output

scripts/llvm-dependencies.yaml

@@ -70,19 +70,21 @@ allprojects:
 
 # projects excluded from automatic configuration as they could not be built
 excludedProjects:
-  # These projects are not working with Visual Studio Compiler on Windows
   windows:
     - lldb
     - llgo
     - libunwind
-    - libcxxabi
+    - libcxx  # no windows support
-    - openmp # blacklisting as kuhnel has trouble with the Perl installation
+    - libcxxabi  # no windows support
+    - openmp # TODO: check: kuhnel has trouble with the Perl installation
     - debuginfo-tests # test failing
     - polly # test failing
     - flang # compilation failing with invalid compile arguments
     # test stuck, needs to be killed manually: instrprof-multiprocess.test
     - compiler-rt
   linux:
+    - libcxx  # has custom checks
+    - libcxxabi  # has custom checks
     # tests keep failing:
     # debuginfo-tests :: llgdb-tests/asan-deque.cpp
     # debuginfo-tests :: llgdb-tests/asan.c

scripts/pipeline_master.py

@@ -15,9 +15,13 @@
 
 import os
 
-from steps import generic_linux, generic_windows
+from steps import generic_linux, generic_windows, from_shell_output
 import yaml
 
+steps_generators = [
+    '${BUILDKITE_BUILD_CHECKOUT_PATH}/libcxx/utils/ci/buildkite-pipeline-snapshot.sh',
+]
+
 if __name__ == '__main__':
     scripts_refspec = os.getenv("ph_scripts_refspec", "master")
     no_cache = os.getenv('ph_no_cache') is not None
@@ -30,9 +34,15 @@ if __name__ == '__main__':
         os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;openmp;polly'),
         False))
     # FIXME: openmp is removed as it constantly fails.
-    # TODO: Make this project list be evaluated through "choose_projects".
+
+    # TODO: Make this project list be evaluated through "choose_projects"(? as now we define "all" and exclusions in
+    #  two placess).
     steps.extend(generic_windows(
         os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly')))
+
+    for gen in steps_generators:
+        steps.extend(from_shell_output(gen))
+
     notify = []
     for e in notify_emails:
         notify.append({'email': e})

scripts/pipeline_premerge.py

@@ -18,9 +18,13 @@ import os
 
 from choose_projects import ChooseProjects
 import git
-from steps import libcxx, generic_linux, generic_windows
+from steps import generic_linux, generic_windows, from_shell_output
 import yaml
 
+steps_generators = [
+    '${BUILDKITE_BUILD_CHECKOUT_PATH}/libcxx/utils/ci/buildkite-pipeline-premerge.sh',
+]
+
 if __name__ == '__main__':
     scripts_refspec = os.getenv("ph_scripts_refspec", "master")
     diff_id = os.getenv("ph_buildable_diff", "")
@@ -42,33 +46,18 @@ if __name__ == '__main__':
     logging.info(f'modified projects: {modified_projects}')
     # Add projects that depend on modified.
     affected_projects = cp.get_affected_projects(modified_projects)
-
-    # Handle special checks.
-    checked = set()
-    generic_projects = set()
-    logging.info(f'all affected projects: {affected_projects}')
     steps = []
-    for p in affected_projects:
+    projects = cp.add_dependencies(affected_projects)
-        if p == 'libcxx' or p == 'libcxxabi':
+    logging.info(f'projects with dependencies: {projects}')
-            if 'libcxx' not in checked:
+    excluded_linux = cp.get_excluded('linux')
-                logging.info('Adding custom steps for "libc++"')
+    logging.info(f'excluded for linux: {excluded_linux}')
-                checked.add('libcxx')
+    steps.extend(generic_linux(';'.join(sorted(projects - excluded_linux)), True))
-                steps.extend(libcxx())
+    excluded_windows = cp.get_excluded('windows')
-        else:
+    logging.info(f'excluded for windows: {excluded_windows}')
-            generic_projects.add(p)
+    steps.extend(generic_windows(';'.join(sorted(projects - excluded_windows))))
 
-    if len(generic_projects) > 0:
+    for gen in steps_generators:
-        # Add dependencies
+        steps.extend(from_shell_output(gen))
-        projects = cp.add_dependencies(generic_projects)
-        logging.info(f'all projects {projects}')
-        excluded_linux = cp.get_excluded('linux')
-        logging.info(f'excluded for linux: {excluded_linux}')
-        steps.extend(generic_linux(';'.join(sorted(projects - excluded_linux)), True))
-        excluded_windows = cp.get_excluded('windows')
-        logging.info(f'excluded for windows: {excluded_windows}')
-        steps.extend(generic_windows(';'.join(sorted(projects - excluded_windows))))
-    else:
-        logging.info('No projects for default checks')
 
     if os.getenv('ph_target_phid') is None:
         logging.warning('ph_target_phid is not specified. Skipping "Report" step')
@@ -99,4 +88,5 @@ if __name__ == '__main__':
             'timeout_in_minutes': 10,
         }
         steps.append(report_step)
+        
     print(yaml.dump({'steps': steps}))

scripts/steps.py

@@ -13,14 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import io
 import json
+import logging
 import os
 from typing import List
 
-
+from exec_utils import watch_shell
-def libcxx():
+import yaml
-    # TODO: read steps from the YAML file from the repo.
-    return []
 
 
 def generic_linux(projects: str, check_diff: bool) -> List:
@@ -143,3 +143,34 @@ def generic_windows(projects: str) -> List:
         ]},
     }
     return [windows_buld_step]
+
+
+def from_shell_output(command) -> []:
+    """
+    Executes shell command and parses stdout as multidoc yaml file, see
+    https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
+    :param command: command, may include env variables
+    :return: all 'steps' that defined in the result ("env" section is ignored).
+             Non-zero exit code and malformed YAML produces empty result.
+    """
+    path = os.path.expandvars(command)
+    logging.debug(f'invoking "{path}"')
+    out = io.BytesIO()
+    err = io.BytesIO()
+    rc = watch_shell(out.write, err.write, path)
+    logging.debug(f'exit code: {rc}, stdout: "{out.getvalue().decode()}", stderr: "{err.getvalue().decode()}"')
+    steps = []
+    if rc != 0:
+        logging.error(
+            f'{path} returned non-zero code {rc}, stdout: "{out.getvalue().decode()}", stderr: "{err.getvalue().decode()}"')
+        return steps
+    try:
+        for part in yaml.safe_load_all(out.getvalue()):
+            part.setdefault('steps', [])
+            steps.extend(part['steps'])
+    except yaml.YAMLError as e:
+        logging.error(f'''"{path}" produced malformed YAML, exception:
+{e}
+
+stdout: >>>{out.getvalue().decode()}>>>''')
+    return steps