Merge branch 'master' of ssh://github.com/google/llvm-premerge-checks

.gitignore

@@ -1,3 +1,5 @@
 .vscode/
 .idea/
 *.iml
+venv/
+__pycache__/

Jenkins/Phabricator-pipeline/Jenkinsfile

@@ -46,16 +46,25 @@ pipeline {
                     currentBuild.displayName += " D${drev_id}"
                     currentBuild.description = "<a href='https://reviews.llvm.org/D${drev_id}'>D${drev_id}</a>"
                 }
-                // Report versions of the installed packages.
+                 // Report versions of the installed packages.
-                sh 'dpkg -l'
+                 sh '''
-                sh 'clang --version'
+                    echo Versions of various tools:
-                sh 'python --version'
+                    echo clang --version
+                    clang --version
+                    echo clang-tidy --version
+                    clang-tidy --version
+                    echo ld.lld --version
+                    ld.lld --version
+                    dpkg -l
+                    '''
+                
             }
         }
         stage("git checkout"){
             steps {
                 git url: 'https://github.com/llvm/llvm-project.git'
                 sh 'git clean -fdx'
+                sh 'git show -s'
                 sh 'mkdir -p llvm-premerge-checks'
                 dir("llvm-premerge-checks")
                 {
@@ -149,6 +158,7 @@ pipeline {
                 --buildresult ${currentBuild.result} \
                 --clang-format-patch "clang-format.patch" \
                 --clang-tidy-result "clang-tidy.txt" \
+                --clang-tidy-ignore "${SCRIPT_DIR}/clang-tidy-comments.ignore" \
                 --results-dir "${TARGET_DIR}" \
                 --results-url "${RESULT_URL}"
                 """

Jenkins/Phabricator-windows-pipeline/Jenkinsfile

@@ -48,6 +48,8 @@ pipeline {
                 echo "getting llvm-project... "
                 dir("llvm-project")
                 {
+                    bat ""
+                    bat 'if exist "build" rd /s/q "build"'
                     git url: 'https://github.com/llvm/llvm-project.git'
                 }
                 powershell "New-Item -ItemType Directory -Force -Path ${RESULT_DIR}"
@@ -91,15 +93,17 @@ pipeline {
                 // gather all result files in a folder, then upload everything to
                 // Google Cloud Storage
                 powershell """
-                    # get the console log
+                    Write-Host "Getting the console log..."
                     Invoke-WebRequest -OutFile console-log.txt -uri "http://jenkins.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText" -ErrorAction "Continue"
-
-                    Copy-Item "${LLVM_DIR}\\build\\CMakeCache.txt"
-                    Copy-Item "${LLVM_DIR}\\build\\test-results.xml"
                     
-                    # upload files
+                    Write-Host "Copying build artefacts..."
+                    Copy-Item "${LLVM_DIR}\\build\\CMakeCache.txt" -ErrorAction SilentlyContinue
+                    Copy-Item "${LLVM_DIR}\\build\\test-results.xml" -ErrorAction SilentlyContinue
+                    
+                    Write-Host "Uploading results to GCS..."
                     \$ErrorActionPreference = 'SilentlyContinue'
                     gsutil cp -Z *.* gs://llvm-premerge-checks/results/${MY_BUILD_ID}/
+                    Write-Host "Done."
                 """
             }
             // doesn't find junit results, not sure why...

containers/agent-debian-testing-buildkite/Dockerfile

@@ -1,57 +1,61 @@
-FROM debian:testing
+FROM debian:testing
-
+
-RUN apt-get update ;\
+RUN echo "deb [trusted=yes] http://apt.llvm.org/buster/ llvm-toolchain-buster-10 main\n$(cat /etc/apt/sources.list)" > /etc/apt/sources.list ;\
-    apt-get install -y --no-install-recommends locales \
+    apt-get update ;\
-        cmake ninja-build git ca-certificates clang lld ccache python3 build-essential \
+    apt-get install -y --no-install-recommends locales \
-        clang-tidy clang-format \
+        cmake ninja-build git ca-certificates clang lld ccache python3 build-essential \
-        python3-psutil zip wget \
+        clang-tidy clang-format \
-        openjdk-11-jdk \
+        python3-psutil zip wget \
-        python3-pip python3-setuptools \
+        openjdk-11-jdk \
-        swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq; \
+        python3-pip python3-setuptools \
-        apt-get clean
+        swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq; \
-
+        apt-get clean
-# Make python3 default (needed by git-clang-format and others).
+
-RUN rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python
+# Make python3 default (needed by git-clang-format and others).
-# required for openssh server
+RUN rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python
-RUN mkdir -p /run/sshd
+RUN ls /usr/bin
-
+# required for openssh server
-ARG user=buildkite
+RUN mkdir -p /run/sshd
-ARG group=buildkite
+
-ARG uid=1000
+ARG user=buildkite
-ARG gid=1000
+ARG group=buildkite
-ARG AGENT_WORKDIR=/home/${user}/agent
+ARG uid=1000
-
+ARG gid=1000
-RUN mkdir -p /scripts
+ARG AGENT_WORKDIR=/home/${user}/agent
-
+
-COPY start_agent.sh /scripts/
+RUN mkdir -p /scripts
-
+
-# install python dependencies for the scripts
+COPY start_agent.sh /scripts/
-RUN pip3 install -r https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/phabtalk/requirements.txt
+
-
+# install python dependencies for the scripts
-RUN groupadd -g ${gid} ${group} ;\
+# ADD will checks that contentent of a file has changed.
-    useradd -c "buildkite user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} ;\
+ADD "https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/phabtalk/requirements.txt" requirements.txt
-    mkdir /home/${user}/ccache
+RUN pip3 install -r requirements.txt
-
+
-RUN chown -R ${user}:${user} /home/${user}
+RUN groupadd -g ${gid} ${group} ;\
-
+    useradd -c "buildkite user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} ;\
-WORKDIR /home/${user}
+    mkdir /home/${user}/ccache
-ENV CCACHE_PATH=/mnt/disks/ssd0/ccache
+
-
+RUN chown -R ${user}:${user} /home/${user}
-# configure locale
+
-RUN sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\
+WORKDIR /home/${user}
-    locale-gen
+ENV CCACHE_PATH=/mnt/disks/ssd0/ccache
-ENV LANG en_US.UTF-8  
+
-ENV LANGUAGE en_US:en  
+# configure locale
-ENV LC_ALL en_US.UTF-8  
+RUN sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\
-
+    locale-gen
-# buildkite
+ENV LANG en_US.UTF-8  
-RUN apt-get install -y apt-transport-https gnupg;\
+ENV LANGUAGE en_US:en  
-    sh -c 'echo deb https://apt.buildkite.com/buildkite-agent stable main > /etc/apt/sources.list.d/buildkite-agent.list' ;\
+ENV LC_ALL en_US.UTF-8  
-    apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198 ;\
+
-    apt-get update ;\
+# buildkite
-    apt-get install -y buildkite-agent
+RUN apt-get install -y apt-transport-https gnupg;\
-
+    sh -c 'echo deb https://apt.buildkite.com/buildkite-agent stable main > /etc/apt/sources.list.d/buildkite-agent.list' ;\
-VOLUME /credentials
+    apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198 ;\
-
+    apt-get update ;\
-CMD ["/scripts/start_agent.sh"]
+    apt-get install -y buildkite-agent
-
+
+VOLUME /credentials
+
+CMD ["/scripts/start_agent.sh"]
+

containers/agent-debian-testing-ssd/Dockerfile

@@ -1,53 +1,56 @@
-FROM debian:testing
+FROM debian:testing
-
+
-RUN apt-get update ;\
+RUN echo "deb [trusted=yes] http://apt.llvm.org/buster/ llvm-toolchain-buster-10 main\n$(cat /etc/apt/sources.list)" > /etc/apt/sources.list;\
-    apt-get install -y --no-install-recommends locales \
+    apt-get update ;\
-        cmake ninja-build git ca-certificates clang lld ccache python3 build-essential \
+    apt-get install -y --no-install-recommends locales \
-        clang-tidy clang-format \
+        cmake ninja-build git ca-certificates clang lld ccache python3 build-essential \
-        python3-psutil arcanist zip wget \
+        clang-tidy clang-format \
-        openjdk-11-jdk \
+        python3-psutil arcanist zip wget \
-        python3-pip python3-setuptools \
+        openjdk-11-jdk \
-        swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq; \
+        python3-pip python3-setuptools \
-        apt-get clean
+        swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq; \
-
+        apt-get clean
-# Make python3 default (needed by git-clang-format and others).
+
-RUN rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python
+# Make python3 default (needed by git-clang-format and others).
-# required for openssh server
+RUN rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python
-RUN mkdir -p /run/sshd
+# required for openssh server
-
+RUN mkdir -p /run/sshd
-ARG user=jenkins
+
-ARG group=jenkins
+ARG user=jenkins
-ARG uid=1000
+ARG group=jenkins
-ARG gid=1000
+ARG uid=1000
-ARG AGENT_WORKDIR=/home/${user}/agent
+ARG gid=1000
-
+ARG AGENT_WORKDIR=/home/${user}/agent
-RUN mkdir -p /scripts
+
-
+RUN mkdir -p /scripts
-# install the swarm client
+
-RUN cd /scripts ;\
+# install the swarm client
-    wget https://repo.jenkins-ci.org/releases/org/jenkins-ci/plugins/swarm-client/3.17/swarm-client-3.17.jar ;\
+RUN cd /scripts ;\
-    mv swarm-client-3.17.jar swarm-client.jar 
+    wget https://repo.jenkins-ci.org/releases/org/jenkins-ci/plugins/swarm-client/3.17/swarm-client-3.17.jar ;\
-
+    mv swarm-client-3.17.jar swarm-client.jar 
-COPY start_agent.sh report_results.sh /scripts/
+
-
+COPY start_agent.sh report_results.sh /scripts/
-# install python dependencies for the scripts
+
-RUN pip3 install -r https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/phabtalk/requirements.txt
+# install python dependencies for the scripts
-
+# ADD will checks that contentent of a file has changed.
-RUN groupadd -g ${gid} ${group} ;\
+ADD "https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/phabtalk/requirements.txt" requirements.txt
-    useradd -c "Jenkins user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} ;\
+RUN pip3 install -r requirements.txt
-    mkdir /home/${user}/ccache
+
-
+RUN groupadd -g ${gid} ${group} ;\
-RUN chown -R ${user}:${user} /home/${user}
+    useradd -c "Jenkins user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} ;\
-
+    mkdir /home/${user}/ccache
-WORKDIR /home/${user}
+
-ENV CCACHE_PATH=/mnt/disks/ssd0/ccache
+RUN chown -R ${user}:${user} /home/${user}
-
+
-# configure locale
+WORKDIR /home/${user}
-RUN sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\
+ENV CCACHE_PATH=/mnt/disks/ssd0/ccache
-    locale-gen
+
-ENV LANG en_US.UTF-8  
+# configure locale
-ENV LANGUAGE en_US:en  
+RUN sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\
-ENV LC_ALL en_US.UTF-8  
+    locale-gen
-
+ENV LANG en_US.UTF-8  
-CMD ["/scripts/start_agent.sh"]
+ENV LANGUAGE en_US:en  
-
+ENV LC_ALL en_US.UTF-8  
+
+CMD ["/scripts/start_agent.sh"]
+

containers/build_deploy.sh

@@ -25,6 +25,6 @@ IMAGE_NAME="${1%/}"
 QUALIFIED_NAME="${GCR_HOSTNAME}/${GCP_PROJECT}/${IMAGE_NAME}"
 
 cd "${DIR}/${IMAGE_NAME}"
-docker build -t ${IMAGE_NAME} .
+docker build --no-cache -t ${IMAGE_NAME} .
 docker tag ${IMAGE_NAME} ${QUALIFIED_NAME}
-docker push ${QUALIFIED_NAME} 
+docker push ${QUALIFIED_NAME}

containers/build_run.sh

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 # Starts a new instances of a docker image. Example:
-# sudo build_run.sh agent-debian-testing-clang8-ssd /bin/bash
+# sudo build_run.sh agent-debian-testing-ssd /bin/bash
 
 set -eux
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

docs/clang_tidy.md

@@ -0,0 +1,29 @@
+# clang-tidy checks
+## Warning is not useful
+If you found that a warning produced by clang-tidy is not useful:
+  
+- If clang-tidy must not run for some files at all (e.g. lit test), please
+[add files to blacklist](../scripts/clang-tidy.ignore).
+
+- Consider fixing or [suppressing diagnostic](https://clang.llvm.org/extra/clang-tidy/#suppressing-undesired-diagnostics)
+  if there is a good reason.
+  
+- [File a bug](https://github.com/google/llvm-premerge-checks/issues/new?assignees=&labels=bug&template=bug_report.md&title=)
+  if build process should be improved. 
+
+- If you believe that you found a clang-tidy bug then please keep in mind that clang-tidy version run by bot
+  might not be most recent. Please reproduce your issue on current version before submitting a bug to clang-tidy.
+
+## Review comments
+
+Build bot leaves inline comments only for a small subset of files that are not blacklisted for analysis (see above) *and*
+specifically whitelisted for comments.
+
+That is done to avoid potential noise when a file already contains a number of warnings.
+
+If your are confident that some files are in good shape already, please
+[whitelist them](../scripts/clang-tidy-comments.ignore).
+
+----
+
+[about pre-merge checks](docs/user_doc.md)

docs/playbooks.md

@@ -6,11 +6,14 @@
 General remarks:
 * GCP does not route any traffic to your services unless the service is "healthy". It might take a few minutes after startup before the services is classified as healthy. Until then you will only see some generic error message.
 
-These are the steps to set up the build server on a clean infrastructure:
+These are the steps to set up the build server on a clean infrastructure:  
 1. Configure the tools on your local machine:
     ```bash
     ./local_setup.sh
     ```
+   If you not running docker under your user, you might need to
+   `sudo gcloud auth login --no-launch-browser && gcloud auth configure-docker`
+   before running other commands under sudo.
 1. Delete the old cluster, if it still exists:
     ```bash
     cd kubernetes/cluster
@@ -120,13 +123,21 @@ To spawn a new windows agent:
 1. In the RDP session: run these commands in the CMD window to start the docker container:
 ```cmd
 powershell 
-Invoke-WebRequest -uri 'https://raw.githubusercontent.com/google/llvm-premerge-checks/master/kubernetes/windows_agent_bootstrap.ps1' -OutFile windows_agent_bootstrap.ps1
+Invoke-WebRequest -uri 'https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/windows_agent_bootstrap.ps1' -OutFile windows_agent_bootstrap.ps1
+.\windows_agent_bootstrap.ps1
+```
+1. Wait for the machine to reboot, then login again and store the `gsutil` credentials in `build-agent-results_key`.
+TODO: add documentation on how to create these.
+1. run this script to start containers:
+```cmd
+powershell 
+Invoke-WebRequest -uri 'https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/windows_agent_start.ps1' -OutFile windows_agent_bootstrap.ps1
 .\windows_agent_bootstrap.ps1
 ```
 
 ## Testing scripts locally
 
-Build and run agent docker image `sudo ./containers/build_run.sh agent-debian-testing-clang8-ssd /bin/bash`.
+Build and run agent docker image `sudo ./containers/build_run.sh agent-debian-testing-ssd /bin/bash`.
 
 Within a container set environment variables similar to [pipeline](https://github.com/google/llvm-premerge-checks/blob/master/Jenkins/Phabricator-pipeline/Jenkinsfile).
 

docs/user_doc.md

@@ -44,17 +44,19 @@ Only then can the build server apply the patch locally and run the builds and te
 Once you're signed up, Phabricator will automatically trigger a build for every new patch you upload or every existing patch you modify. Phabricator shows the build results at the top of the entry:
 ![build status](images/diff_detail.png)
 
+Bot will compile and run tests, run clang-format and [clang-tidy](docs/clang_tidy.md) on lines changed. 
+
 If a unit test failed, this is shown below the build status. You can also expand the unit test to see the details:
 ![unit test results](images/unit_tests.png)
 
-After every build the build server will comment on your latest patch, so that you can also see the results for previous changes. The comment also contains a link to the log files:
+After every build the build server will comment on your latest patch, so that you can also see the results for previous changes.
+The comment also contains a link to the log files:
 ![bot comment](images/bot_comment.png)
 
 The build logs are stored for 90 days and automatically deleted after that.
 
 You can also trigger a build manually by using the "Run Plan Manually" link on the [Harbormaster page](https://reviews.llvm.org/harbormaster/plan/3/) and entering a revision ID in the pop-up window.
 
-
 # Reporting issues
 
 If you notice any bugs, please create a [new issue](https://github.com/google/llvm-premerge-checks/issues/new).

kubernetes/jenkins.yaml

@@ -151,18 +151,18 @@ spec:
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
-  name: agent-debian-testing-clang8-ssd
+  name: agent-debian-testing-ssd
   namespace: jenkins
 spec:
   replicas: 2
   template:
     metadata:
       labels:
-        app: agent-debian-testing-clang8-ssd
+        app: agent-debian-testing-ssd
     spec:
       containers:
       - name: agent-debian-testing-clang8
-        image: gcr.io/llvm-premerge-checks/agent-debian-testing-clang8-ssd    
+        image: gcr.io/llvm-premerge-checks/agent-debian-testing-ssd
         ports:
         - containerPort: 22
         resources:

scripts/clang-format.ignore

@@ -0,0 +1 @@
+# Patterns for clang-format to ignore.

scripts/clang-tidy-comments.ignore

@@ -0,0 +1,4 @@
+# Files that are allowed by clang-tidy.ignore but should not receive inline review comments.
+# Right now it works in whitelist mode and only some files / directories are whitelisted.
+*
+!clang-tools-extra/clangd/**

scripts/clang-tidy.ignore

@@ -0,0 +1,44 @@
+# Files to be ignored by clang-tidy. Will not appear in short report and inline comments.
+libcxxabi/test
+libcxxabi/test/libcxxabi/test
+libcxx/test
+libcxx/utils/libcxx/test
+libcxx/utils/google-benchmark/test
+llvm/test
+llvm/utils/gn/secondary/llvm/test
+llvm/utils/gn/secondary/lld/test
+llvm/utils/gn/secondary/clang-tools-extra/test
+llvm/utils/gn/secondary/clang-tools-extra/clangd/test
+llvm/utils/gn/secondary/compiler-rt/test
+llvm/utils/gn/secondary/clang/test
+llvm/utils/benchmark/test
+polly/test
+lldb/examples/test
+lldb/test
+lldb/utils/test
+lldb/tools/intel-features/intel-mpx/test
+lldb/packages/Python/lldbsuite/test
+lldb/packages/Python/lldbsuite/test/tools/lldb-server/test
+lldb/packages/Python/lldbsuite/test/commands/expression/test
+lldb/packages/Python/lldbsuite/test/test_runner/test
+lldb/third_party/Python/module/unittest2/unittest2/test
+lld/test
+clang-tools-extra/test
+clang-tools-extra/clangd/clients/clangd-vscode/test
+clang-tools-extra/clangd/test
+pstl/test
+libc/test
+llgo/test
+compiler-rt/test
+compiler-rt/test/builtins/Unit/ppc/test
+compiler-rt/test/builtins/Unit/test
+debuginfo-tests/dexter/dex/tools/test
+debuginfo-tests/dexter/feature_tests/subtools/test
+clang/test
+libclc/test
+mlir/test
+openmp/libomptarget/test
+openmp/libomptarget/deviceRTLs/nvptx/test
+openmp/runtime/test
+libunwind/test
+libunwind/test/libunwind/test

scripts/ignore_diff.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# Copyright 2020 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import sys
+import pathspec
+
+# Takes an output of git diff and removes files ignored by patten specified by ignore file.
+def main():
+    argv = sys.argv[1:]
+    if not argv:
+        print("Please provide a path to .ignore file.")
+        sys.exit(1)
+    ignore = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern,
+                                          open(argv[0], 'r').readlines())
+    good = True
+    for line in sys.stdin:
+        match = re.search(r'^diff --git a/(.*) b/(.*)$', line)
+        if match:
+            good = not (ignore.match_file(match.group(1)) and ignore.match_file(match.group(2)))
+        if not good:
+            continue
+        sys.stdout.write(line)
+
+
+if __name__ == "__main__":
+    main()

scripts/lint.sh

@@ -21,16 +21,17 @@ set -eux
 echo "Running linters... ====================================="
 
 cd "${WORKSPACE}"
+
+# clang-format
 # Let clang format apply patches --diff doesn't produces results in the format we want.
 git-clang-format
 set +e
-git diff -U0 --exit-code > "${TARGET_DIR}"/clang-format.patch
+git diff -U0 --exit-code | "${SCRIPT_DIR}/ignore_diff.py" "${SCRIPT_DIR}/clang-format.ignore" > "${TARGET_DIR}"/clang-format.patch
-STATUS="${PIPESTATUS[0]}"
 set -e
 # Revert changes of git-clang-format.
 git checkout -- .
 
-# TODO: clang tidy is currently disabled, see https://github.com/google/llvm-premerge-checks/issues/91
+# clang-tidy
-# git diff HEAD^ | clang-tidy-diff -p1 -quiet > "${TARGET_DIR}"/clang-tidy.txt
+git diff -U0 HEAD | "${SCRIPT_DIR}/ignore_diff.py" "${SCRIPT_DIR}/clang-tidy.ignore" | clang-tidy-diff -p1 -quiet > "${TARGET_DIR}"/clang-tidy.txt
 
 echo "linters completed ======================================"

scripts/phabtalk/apply_patch.py

@@ -1,4 +1,4 @@
-#!/bin/env python3
+#!/usr/bin/env python3
 # Copyright 2019 Google LLC
 #
 # Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
@@ -87,6 +87,8 @@ class ApplyPatch:
                 'master branch instead...'.format(self.git_hash)]
             subprocess.check_call('git checkout master', stdout=sys.stdout, 
                 stderr=sys.stderr, shell=True)
+        subprocess.check_call('git show -s', stdout=sys.stdout,
+                              stderr=sys.stderr, shell=True)
         print('git checkout completed.')
 
     def _apply_patch(self):

scripts/phabtalk/apply_patch2.py

@@ -21,7 +21,7 @@ import sys
 from typing import List, Optional, Tuple
 
 from phabricator import Phabricator
-from git import Repo
+from git import Repo, GitCommandError
 
 class ApplyPatch:
     """Apply a diff from Phabricator on local working copy.
@@ -76,9 +76,14 @@ class ApplyPatch:
         try:
             revision_id, dependencies, base_revision = self._get_dependencies()
             print('Checking out {}...'.format(base_revision))
-            self.repo.git.checkout(base_revision)
+            try:
+                self.repo.git.checkout(base_revision)
+            except GitCommandError:
+                print('ERROR checking out revision {}. It`s not in the '
+                      'repository. Using master instead.'.format(base_revision))
+                self.repo.git.checkout('master')                
             print('Revision is {}'.format(self.repo.head.commit.hexsha))
-            print('Cleanup...')
+            print('git reset, git cleanup...')
             self.repo.git.reset('--hard')
             self.repo.git.clean('-fdx')
             print('Analyzing {}'.format(diff_to_str(revision_id)))
@@ -139,7 +144,7 @@ class ApplyPatch:
         """Download and apply a diff to the local working copy."""
         print('Applying diff {} for revision {}...'.format(diff_id, diff_to_str(revision_id)))
         diff = self.phab.differential.getrawdiff(diffID=diff_id).response
-        proc = subprocess.run('git apply --whitespace=nowarn --binary -v', input=diff, shell=True, text=True,
+        proc = subprocess.run('patch -p1', input=diff, shell=True, text=True,
             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         if proc.returncode != 0:
             raise Exception('Applying patch failed:\n{}'.format(proc.stdout + proc.stderr))

scripts/phabtalk/phabtalk.py

@@ -22,8 +22,10 @@ import os
 import re
 import socket
 import time
+import urllib
 from typing import Optional
 
+import pathspec
 from lxml import etree
 from phabricator import Phabricator
 
@@ -48,6 +50,7 @@ class BuildReport:
             self.lint[key] = []
         self.lint[key].append(m)
 
+
 class PhabTalk:
     """Talk to Phabricator to upload build results.
        See https://secure.phabricator.com/conduit/method/harbormaster.sendmessage/
@@ -63,7 +66,7 @@ class PhabTalk:
     def dryrun(self):
         return self._phab is None
 
-    def _get_revision_id(self, diff: str):
+    def get_revision_id(self, diff: str):
         """Get the revision ID for a diff from Phabricator."""
         if self.dryrun:
             return None
@@ -75,7 +78,7 @@ class PhabTalk:
         """Add a comment to a differential based on the diff_id"""
         print('Sending comment to diff {}:'.format(diff_id))
         print(text)
-        self._comment_on_revision(self._get_revision_id(diff_id), text)
+        self._comment_on_revision(self.get_revision_id(diff_id), text)
 
     def _comment_on_revision(self, revision: str, text: str):
         """Add comment on a differential based on the revision id."""
@@ -213,18 +216,26 @@ def _add_clang_format(report: BuildReport, results_dir: str,
     report.success = success and report.success
 
 
-def _add_clang_tidy(report: BuildReport, results_dir: str, results_url: str, workspace: str, clang_tidy_file: str):
+def _add_clang_tidy(report: BuildReport, results_dir: str, results_url: str, workspace: str, clang_tidy_file: str,
+                    clang_tidy_ignore: str):
     # Typical message looks like
     # [..]/clang/include/clang/AST/DeclCXX.h:3058:20: error: no member named 'LifetimeExtendedTemporary' in 'clang::Decl' [clang-diagnostic-error]
-
     pattern = '^{}/([^:]*):(\\d+):(\\d+): (.*): (.*)'.format(workspace)
-    success = True
+    errors_count = 0
+    warn_count = 0
+    inline_comments = 0
     present = (clang_tidy_file is not None) and os.path.exists(os.path.join(results_dir, clang_tidy_file))
     if not present:
         print('clang-tidy result {} is not found'.format(clang_tidy_file))
         report.comments.append(section_title('clang-tidy', False, False))
         return
-
+    present = (clang_tidy_ignore is not None) and os.path.exists(clang_tidy_ignore)
+    if not present:
+        print('clang-tidy ignore file {} is not found'.format(clang_tidy_ignore))
+        report.comments.append(section_title('clang-tidy', False, False))
+        return
+    ignore = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern,
+                                          open(clang_tidy_ignore, 'r').readlines())
     p = os.path.join(results_dir, clang_tidy_file)
     for line in open(p, 'r'):
         match = re.search(pattern, line)
@@ -234,21 +245,34 @@ def _add_clang_tidy(report: BuildReport, results_dir: str, results_url: str, wor
             char_pos = match.group(3)
             severity = match.group(4)
             text = match.group(5)
+            text += '\n[[https://github.com/google/llvm-premerge-checks/blob/master/docs/clang_tidy.md#warning-is-not' \
+                    '-useful | not useful]] '
             if severity in ['warning', 'error']:
-                success = False
+                if severity == 'warning':
-                report.addLint({
+                    warn_count += 1
-                    'name': 'clang-tidy',
+                if severity == 'error':
-                    'severity': 'warning',
+                    errors_count += 1
-                    'code': 'clang-tidy',
+                if ignore.match_file(file_name):
-                    'path': file_name,
+                    print('{} is ignored by pattern and no comment will be added'.format(file_name))
-                    'line': int(line_pos),
+                else:
-                    'char': int(char_pos),
+                    inline_comments += 1
-                    'description': '{}: {}'.format(severity, text),
+                    report.addLint({
-                })
+                        'name': 'clang-tidy',
-
+                        'severity': 'warning',
+                        'code': 'clang-tidy',
+                        'path': file_name,
+                        'line': int(line_pos),
+                        'char': int(char_pos),
+                        'description': '{}: {}'.format(severity, text),
+                    })
+    success = errors_count + warn_count == 0
     comment = section_title('clang-tidy', success, present)
     if not success:
-        comment += 'Please fix [[ {}/{} | clang-tidy findings ]].'.format(results_url, clang_tidy_file)
+        comment += "clang-tidy found [[ {}/{} | {} errors and {} warnings]]. {} of them are added as review comments " \
+                   "below ([[https://github.com/google/llvm-premerge-checks/blob/master/docs/clang_tidy.md#review" \
+                   "-comments | why?]])." \
+            .format(results_url, clang_tidy_file, errors_count, warn_count, inline_comments)
+
     report.comments.append(comment)
     report.success = success and report.success
 
@@ -355,10 +379,20 @@ def main():
             report.success = False
 
     _add_test_results(report, args.results_dir, args.test_result_file)
-    _add_clang_tidy(report, args.results_dir, args.results_url, args.workspace, args.clang_tidy_result)
+    _add_clang_tidy(report, args.results_dir, args.results_url, args.workspace, args.clang_tidy_result,
+                    args.clang_tidy_ignore)
     _add_clang_format(report, args.results_dir, args.results_url, args.clang_format_patch)
     _add_links_to_artifacts(report, args.results_dir, args.results_url)
+
     p = PhabTalk(args.conduit_token, args.host, args.dryrun)
+    title = 'Issue with build for {} ({})'.format(p.get_revision_id(args.diff_id), args.diff_id)
+    report.comments.append(
+        '//Pre-merge checks is in beta. [[ https://github.com/google/llvm-premerge-checks/issues/new?assignees'
+        '=&labels=bug&template=bug_report.md&title={} | Report issue]]. '
+        'Please [[ https://reviews.llvm.org/project/update/78/join/ | join beta ]] or '
+        '[[ https://github.com/google/llvm-premerge-checks/issues/new?assignees=&labels=enhancement&template'
+        '=&title=enable%20checks%20for%20{{PATH}} | enable it for your project ]].//'.format(
+            urllib.parse.quote(title)))
     p.submit_report(args.diff_id, args.ph_id, report, args.buildresult)
 
 
@@ -380,6 +414,9 @@ def _parse_args():
     parser.add_argument('--clang-tidy-result', type=str, default=None,
                         dest='clang_tidy_result',
                         help="path to diff produced by git-clang-tidy, relative to results-dir")
+    parser.add_argument('--clang-tidy-ignore', type=str, default=None,
+                        dest='clang_tidy_ignore',
+                        help="path to file with patters to exclude commenting on for clang-tidy findings")
     parser.add_argument('--results-dir', type=str, default=None, required=True,
                         dest='results_dir',
                         help="directory of all build artifacts")

scripts/phabtalk/requirements.txt

@@ -1,4 +1,5 @@
-phabricator==0.7.0
+phabricator==0.7.0
-lxml==4.4.1
+lxml==4.4.1
-gitpython==3.0.5
+gitpython==3.0.5
-retrying==1.3.3
+retrying==1.3.3
+pathspec==0.7.0

scripts/run_cmake.sh

@@ -33,6 +33,7 @@ cmake -GNinja ../llvm -DCMAKE_BUILD_TYPE=Release -D LLVM_ENABLE_LLD=ON \
 RETURN_CODE="${PIPESTATUS[0]}"
 set -e
 
+rm -f "$WORKSPACE/compile_commands.json"
 ln -s "$WORKSPACE"/build/compile_commands.json "$WORKSPACE"
 cp CMakeCache.txt ${TARGET_DIR}
 

scripts/windows_agent_bootstrap.ps1

@@ -12,10 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# 1st stage of the installation process.
+# This script only needs to be run once per machine.
+
 Write-Host "Initializing local SSD..."
 New-Variable -Name diskid -Value (Get-Disk -FriendlyName "Google EphemeralDisk").Number
 # TODO: check if machine has an SSD
-# TODO: only do this, if SSD is not yet usable
+# TODO: only do this, if SSD is not yet partioned and formatted
 Initialize-Disk -Number $diskid
 New-Partition -DiskNumber $diskid -UseMaximumSize -AssignDriveLetter
 Format-Volume -DriveLetter D
@@ -23,11 +26,21 @@ Format-Volume -DriveLetter D
 Write-Host "install chocolately as package manager..."
 iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1'))
 choco feature disable --name showDownloadProgress
-choco install git
+choco install -y git
 
-Write-Host "Mounting result storage..."
+# install Docker
-Install-WindowsFeature NFS-Client
+Install-PackageProvider -Name NuGet -Force
-net use E: \\results.local\exports /PERSISTENT:YES
+Install-Module -Name DockerMsftProvider -Repository PSGallery -Force
+Install-Package -Name docker -ProviderName DockerMsftProvider -Force
+Set-Service -Name docker  -StartupType Automatic
 
-Set-Location D:\
+# install gcloud and authenticate access to gcr.io registry
-git clone https://github.com/google/llvm-premerge-checks
+# TODO: find a better way to install the Google Cloud SDK, avoid ingoring the checksum
+choco install -y gcloudsdk --ignore-checksums
+gcloud auth docker
+
+# move docker folder to SSD to get better IO performance
+cmd /C "mklink /j C:\ProgramData\Docker D:\docker"
+
+# Reboot
+Restart-Computer -Force

scripts/windows_agent_install.ps1

@@ -1,51 +0,0 @@
-# Copyright 2019 Google LLC
-
-# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-
-#     https://llvm.org/LICENSE.txt
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-. ${PSScriptRoot}\common.ps1
-
-Write-Host "Installing Visual Studio build tools..."
-choco install -y visualcpp-build-tools --version 15.0.26228.20170424 -y --params "'/IncludeOptional'"
-Write-Host 'Waiting for Visual C++ Build Tools to finish'
-Wait-Process -Name vs_installer
-
-Write-Host "Installing misc tools"
-# install other tools as described in https://llvm.org/docs/GettingStartedVS.html
-# and a few more that were not documented...
-choco install -y git python2 ninja gnuwin cmake
-pip install psutil
-
-Write-Host "Setting environment variables..."
-[System.Environment]::SetEnvironmentVariable('PYTHONIOENCODING', 'UTF-8', [System.EnvironmentVariableTarget]::User)
-$oldpath=[System.Environment]::GetEnvironmentVariable('path',[System.EnvironmentVariableTarget]::User)
-[System.Environment]::SetEnvironmentVariable('path', $oldpath + 'c:\Program Files (x86)\GnuWin32\bin;C:\Program Files\CMake\bin', [System.EnvironmentVariableTarget]::User)
-# support long file names during git checkout
-Write-Host "Setting git config..."
-git config --system core.longpaths true
-git config --global core.autocrlf false
-
-# Above: genric LLVM-related things
-#-------------
-# Below: Jenkins specific things
-
-Write-Host "Installing openjdk..."
-choco install -y openjdk
-
-Write-Host "Installing Jenkins swarm agent..."
-$SWARM_PLUGIN_URL="https://repo.jenkins-ci.org/releases/org/jenkins-ci/plugins/swarm-client/3.17/swarm-client-3.17.jar"
-$SWARM_PLUGIN_JAR="C:\jenkins\swarm-client.jar"
-mkdir c:\jenkins
-Invoke-WebRequest -Uri $SWARM_PLUGIN_URL -OutFile $SWARM_PLUGIN_JAR
-
-# TODO: put build-agent-results_key.json on the machine somehow
-gcloud auth activate-service-account --key-file C:\jenkins\build-agent-results_key.json

scripts/windows_agent_start.ps1

@@ -1,32 +1,30 @@
 # Copyright 2019 Google LLC
-#
+
 # Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-#
+
 #     https://llvm.org/LICENSE.txt
-#
+
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Pull and start the Docker container for a Windows agent.
+# To setup a Windows agent see docs/playbooks.md
 
-. ${PSScriptRoot}\common.ps1
+$NAME="agent-windows-jenkins"
+$IMAGE="gcr.io/llvm-premerge-checks/${NAME}"
 
-$JENKINS_SERVER="jenkins.local"
+docker pull ${IMAGE}
-$AGENT_ROOT="D:\"
+docker stop ${NAME}
-$SWARM_PLUGIN_JAR="C:\jenkins\swarm-client.jar"
+docker rm ${NAME}
-
+docker run `
-# move temp dir to local SSD
+    -v D:\:C:\ws `
-mkdir D:\temp
+    -v C:\credentials:C:\credentials `
-$env:temp="D:\temp"
+    -e PARENT_HOSTNAME=$env:computername `
-$env:tmp=$env:temp
+    --restart unless-stopped `
-
+    --name ${NAME} `
-java -jar ${SWARM_PLUGIN_JAR} `
+    ${IMAGE}
-    -master http://${JENKINS_SERVER}:8080 `
-    -executors 1 `
-    -fsroot ${AGENT_ROOT} `
-    -labels windows `
-    -name $env:computername