Merge branch 'master' of ssh://github.com/google/llvm-premerge-checks

Jenkins/Phabricator-pipeline/Jenkinsfile

@@ -174,7 +174,8 @@ pipeline {
                 --clang-tidy-ignore "${SCRIPT_DIR}/clang-tidy-comments.ignore" \
                 --results-dir "${TARGET_DIR}" \
                 --results-url "${RESULT_URL}" \
-                --failures "${failure_message}"
+                --failures "${failure_message}" \
+                --name "linux"
                 """
         }
     }    

Jenkins/Phabricator-windows-pipeline/Jenkinsfile

@@ -25,12 +25,15 @@ pipeline {
         PHABRICATOR_HOST = 'https://reviews.llvm.org'
         PHAB_LOG = "${WORKSPACE}/build/.phabricator-comment"
         MY_BUILD_ID = "${JOB_BASE_NAME}-${BUILD_NUMBER}"
-        RESULT_URL = "https://pantheon.corp.google.com/storage/browser/llvm-premerge-checks/results/${MY_BUILD_ID}"
+        RESULT_URL = "https://storage.cloud.google.com/llvm-premerge-checks/results/${MY_BUILD_ID}"
         SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts"
         // store all build results here, will be uploaded to GCS later
         RESULT_DIR = "${WORKSPACE}\\results"
         LLVM_DIR = "${WORKSPACE}\\llvm-project"
     }
+    options {
+        timeout(time:2, unit:'HOURS')
+    }
     stages {
         stage("build info"){
             steps {
@@ -159,7 +162,8 @@ pipeline {
                     --results-dir "${RESULT_DIR}" ^
                     --results-url "${RESULT_URL}" ^
                     --failures "${failure_message}" ^
-                    --buildresult ${currentBuild.result}
+                    --buildresult ${currentBuild.result} ^
+                    --name "windows"
                 """            
             dir("${RESULT_DIR}") {
                 // upload results to

scripts/lint.sh

@@ -32,6 +32,6 @@ set -e
 git checkout -- .
 
 # clang-tidy
-git diff -U0 HEAD | "${SCRIPT_DIR}/ignore_diff.py" "${SCRIPT_DIR}/clang-tidy.ignore" | clang-tidy-diff -p1 -quiet > "${TARGET_DIR}"/clang-tidy.txt
+git diff -U0 HEAD | "${SCRIPT_DIR}/ignore_diff.py" "${SCRIPT_DIR}/clang-tidy.ignore" | clang-tidy-diff -p1 -quiet | sed '${/^[[:space:]]*$/d;}' > "${TARGET_DIR}"/clang-tidy.txt
 
 echo "linters completed ======================================"

scripts/metrics/repo.py

@@ -16,7 +16,7 @@
 import datetime
 from time import timezone
 import git
-from typing import Dict
+from typing import Dict, Optional
 from google.cloud import monitoring_v3
 import re
 
@@ -32,21 +32,35 @@ class RepoStats:
         self.reviewed = 0  # type: int
 
     @property
-    def percent_reverted(self) -> float:
+    def percent_reverted(self) -> Optional[float]:
-        return 100.0 * self.reverts / self.commits
+        try:
+            return 100.0 * self.reverts / self.commits
+        except ZeroDivisionError:
+            return None
 
     @property
-    def percent_reviewed(self) -> float:
+    def percent_reviewed(self) -> Optional[float]:
-        return 100.0 * self.reviewed / (self.commits - self.reverts)
+        try:
+            return 100.0 * self.reviewed / (self.commits - self.reverts)
+        except ZeroDivisionError:
+            return None
 
     def __str__(self):
-        return "\n".join([
+        results = [
             "commits:  {}".format(self.commits),
             "reverts:  {}".format(self.reverts),
             "reviewed: {}".format(self.reviewed),
-            "percent reverted: {:0.1f}".format(self.percent_reverted),
+        ]
-            "percent reviewed: {:0.1f}".format(self.percent_reviewed),
+        try:
-            ])
+            results.append("percent reverted: {:0.1f}".format(self.percent_reverted))
+        except TypeError:
+            pass
+        try:
+            results.append("percent reverted: {:0.1f}".format(self.percent_reverted))
+        except TypeError:
+            pass
+
+        return "\n".join(results)
 
 def get_reverts_per_day(repo_path: str, max_age: datetime.datetime) -> RepoStats:
     stats = RepoStats()
@@ -78,6 +92,9 @@ def gcp_write_data(project_id: str, stats: RepoStats, now:datetime.datetime):
         ["reviewed", stats.reviewed],
         ["percent_reviewed", stats.percent_reviewed],
     ]:
+        if value is None:
+            continue
+
         series = monitoring_v3.types.TimeSeries()
         series.metric.type = 'custom.googleapis.com/repository_{}'.format(desc_type)
         series.resource.type = 'global'

scripts/phabtalk/phabtalk.py

@@ -40,7 +40,7 @@ class PhabTalk:
         self._phab = None  # type: Optional[Phabricator]
         if not dryrun:
             self._phab = Phabricator(token=token, host=host)
-            self._phab.update_interfaces()
+            _try_call(self._phab.update_interfaces)
 
     @property
     def dryrun(self):
@@ -124,9 +124,9 @@ class PhabTalk:
             lint=lint_messages))
 
     def add_artifact(self, phid: str, file: str, name: str, results_url: str):
-        artifactKey=str(uuid.uuid4())
+        artifactKey = str(uuid.uuid4())
-        artifactType='uri'
+        artifactType = 'uri'
-        artifactData={'uri': '{}/{}'.format(results_url, file),
+        artifactData = {'uri': '{}/{}'.format(results_url, file),
                         'ui.external': True,
                         'name': name}
         if self.dryrun:
@@ -203,9 +203,11 @@ class BuildReport:
         self.results_url = args.results_url  # type: str
         self.workspace = args.workspace  # type: str
         self.failure_messages = args.failures  # type: str
+        self.name = args.name  # type: str
 
         self.api = PhabTalk(args.conduit_token, args.host, args.dryrun)
 
+        self.revision_id = self.api.get_revision_id(self.diff_id)
         self.comments = []
         self.success = True
         self.working = False
@@ -233,7 +235,16 @@ class BuildReport:
         else:
             self.success = False
 
-        self.add_test_results()
+        try:
+            self.add_test_results()
+        except etree.XMLSyntaxError:
+            # Sometimes we get an incomplete XML file.
+            # In this case: 
+            #   - fail the build (the safe thing to do)
+            #   - continue so the user gets some feedback.
+            print('Error parsing {}. Invalid XML syntax!'.format(self.test_result_file))
+            self.success = False
+
         self.add_clang_tidy()
         self.add_clang_format()
         self.api.update_build_status(self.diff_id, self.ph_id, self.working, self.success, self.lint, self.unit)
@@ -254,12 +265,14 @@ class BuildReport:
                     '.failure {color:red;}\n'
                     '.success {color:green;}\n'
                     '</style></head><body>')
+            f.write('<h1>Build result for diff <a href="https://reviews.llvm.org/{0}">{0}</a> {1} at {2}</h1>'.format(
+                self.revision_id, self.diff_id, self.name))
             if self.failure_messages and len(self.failure_messages) > 0:
                 for s in self.failure_messages.split('\n'):
                     f.write('<p class="failure">{}</p>'.format(s))
             f.write('<p>' + '</p><p>'.join(self.comments) + '</p>')
             f.write('</body></html>')
-            self.api.add_artifact(self.ph_id, 'summary.html', 'summary', self.results_url)
+            self.api.add_artifact(self.ph_id, 'summary.html', 'summary ' + self.name, self.results_url)
 
     def add_clang_format(self):
         """Populates results from diff produced by clang format."""
@@ -273,7 +286,7 @@ class BuildReport:
             return
         p = os.path.join(self.results_dir, self.clang_format_patch)
         if os.stat(p).st_size != 0:
-            self.api.add_artifact(self.ph_id, self.clang_format_patch, "clang-format", self.results_url)
+            self.api.add_artifact(self.ph_id, self.clang_format_patch, 'clang-format ' + self.name, self.results_url)
         diffs = _parse_patch(open(p, 'r'))
         success = len(diffs) == 0
         for d in diffs:
@@ -314,8 +327,8 @@ class BuildReport:
             self.comments.append(section_title('clang-tidy', False, False))
             return
         p = os.path.join(self.results_dir, self.clang_tidy_result)
-        if os.stat(p).st_size > 4:
+        if os.stat(p).st_size > 0:
-            self.api.add_artifact(self.ph_id, self.clang_tidy_result, "clang-tidy", self.results_url)
+            self.api.add_artifact(self.ph_id, self.clang_tidy_result, 'clang-tidy ' + self.name, self.results_url)
         ignore = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern,
                                               open(self.clang_tidy_ignore, 'r').readlines())
         for line in open(p, 'r'):
@@ -482,6 +495,7 @@ def main():
                         help="public URL to access results directory")
     parser.add_argument('--workspace', type=str, required=True, help="path to workspace")
     parser.add_argument('--failures', type=str, default=None, help="optional failure messages separated by newline")
+    parser.add_argument('--name', type=str, default='', help="optional name of the build bot")
     args = parser.parse_args()
 
     reporter = BuildReport(args)