diff --git a/.gitignore b/.gitignore index 238546f..21455c2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .vscode/ .idea/ *.iml +venv/ +__pycache__/ diff --git a/Jenkins/Phabricator-pipeline/Jenkinsfile b/Jenkins/Phabricator-pipeline/Jenkinsfile index c02bc84..23f071d 100644 --- a/Jenkins/Phabricator-pipeline/Jenkinsfile +++ b/Jenkins/Phabricator-pipeline/Jenkinsfile @@ -46,16 +46,25 @@ pipeline { currentBuild.displayName += " D${drev_id}" currentBuild.description = "D${drev_id}" } - // Report versions of the installed packages. - sh 'dpkg -l' - sh 'clang --version' - sh 'python --version' + // Report versions of the installed packages. + sh ''' + echo Versions of various tools: + echo clang --version + clang --version + echo clang-tidy --version + clang-tidy --version + echo ld.lld --version + ld.lld --version + dpkg -l + ''' + } } stage("git checkout"){ steps { git url: 'https://github.com/llvm/llvm-project.git' sh 'git clean -fdx' + sh 'git show -s' sh 'mkdir -p llvm-premerge-checks' dir("llvm-premerge-checks") { @@ -149,6 +158,7 @@ pipeline { --buildresult ${currentBuild.result} \ --clang-format-patch "clang-format.patch" \ --clang-tidy-result "clang-tidy.txt" \ + --clang-tidy-ignore "${SCRIPT_DIR}/clang-tidy-comments.ignore" \ --results-dir "${TARGET_DIR}" \ --results-url "${RESULT_URL}" """ diff --git a/Jenkins/Phabricator-windows-pipeline/Jenkinsfile b/Jenkins/Phabricator-windows-pipeline/Jenkinsfile index d8c75f4..1b0093a 100644 --- a/Jenkins/Phabricator-windows-pipeline/Jenkinsfile +++ b/Jenkins/Phabricator-windows-pipeline/Jenkinsfile @@ -48,6 +48,8 @@ pipeline { echo "getting llvm-project... " dir("llvm-project") { + bat "" + bat 'if exist "build" rd /s/q "build"' git url: 'https://github.com/llvm/llvm-project.git' } powershell "New-Item -ItemType Directory -Force -Path ${RESULT_DIR}" @@ -91,15 +93,17 @@ pipeline { // gather all result files in a folder, then upload everything to // Google Cloud Storage powershell """ - # get the console log + Write-Host "Getting the console log..." Invoke-WebRequest -OutFile console-log.txt -uri "http://jenkins.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText" -ErrorAction "Continue" - - Copy-Item "${LLVM_DIR}\\build\\CMakeCache.txt" - Copy-Item "${LLVM_DIR}\\build\\test-results.xml" - # upload files + Write-Host "Copying build artefacts..." + Copy-Item "${LLVM_DIR}\\build\\CMakeCache.txt" -ErrorAction SilentlyContinue + Copy-Item "${LLVM_DIR}\\build\\test-results.xml" -ErrorAction SilentlyContinue + + Write-Host "Uploading results to GCS..." \$ErrorActionPreference = 'SilentlyContinue' gsutil cp -Z *.* gs://llvm-premerge-checks/results/${MY_BUILD_ID}/ + Write-Host "Done." """ } // doesn't find junit results, not sure why... diff --git a/containers/agent-debian-testing-clang8-buildkite/Dockerfile b/containers/agent-debian-testing-buildkite/Dockerfile similarity index 80% rename from containers/agent-debian-testing-clang8-buildkite/Dockerfile rename to containers/agent-debian-testing-buildkite/Dockerfile index c2c9bbd..18f6314 100644 --- a/containers/agent-debian-testing-clang8-buildkite/Dockerfile +++ b/containers/agent-debian-testing-buildkite/Dockerfile @@ -1,57 +1,61 @@ -FROM debian:testing - -RUN apt-get update ;\ - apt-get install -y --no-install-recommends locales \ - cmake ninja-build git ca-certificates clang lld ccache python3 build-essential \ - clang-tidy clang-format \ - python3-psutil zip wget \ - openjdk-11-jdk \ - python3-pip python3-setuptools \ - swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq; \ - apt-get clean - -# Make python3 default (needed by git-clang-format and others). -RUN rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python -# required for openssh server -RUN mkdir -p /run/sshd - -ARG user=buildkite -ARG group=buildkite -ARG uid=1000 -ARG gid=1000 -ARG AGENT_WORKDIR=/home/${user}/agent - -RUN mkdir -p /scripts - -COPY start_agent.sh /scripts/ - -# install python dependencies for the scripts -RUN pip3 install -r https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/phabtalk/requirements.txt - -RUN groupadd -g ${gid} ${group} ;\ - useradd -c "buildkite user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} ;\ - mkdir /home/${user}/ccache - -RUN chown -R ${user}:${user} /home/${user} - -WORKDIR /home/${user} -ENV CCACHE_PATH=/mnt/disks/ssd0/ccache - -# configure locale -RUN sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\ - locale-gen -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 - -# buildkite -RUN apt-get install -y apt-transport-https gnupg;\ - sh -c 'echo deb https://apt.buildkite.com/buildkite-agent stable main > /etc/apt/sources.list.d/buildkite-agent.list' ;\ - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198 ;\ - apt-get update ;\ - apt-get install -y buildkite-agent - -VOLUME /credentials - -CMD ["/scripts/start_agent.sh"] - +FROM debian:testing + +RUN echo "deb [trusted=yes] http://apt.llvm.org/buster/ llvm-toolchain-buster-10 main\n$(cat /etc/apt/sources.list)" > /etc/apt/sources.list ;\ + apt-get update ;\ + apt-get install -y --no-install-recommends locales \ + cmake ninja-build git ca-certificates clang lld ccache python3 build-essential \ + clang-tidy clang-format \ + python3-psutil zip wget \ + openjdk-11-jdk \ + python3-pip python3-setuptools \ + swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq; \ + apt-get clean + +# Make python3 default (needed by git-clang-format and others). +RUN rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python +RUN ls /usr/bin +# required for openssh server +RUN mkdir -p /run/sshd + +ARG user=buildkite +ARG group=buildkite +ARG uid=1000 +ARG gid=1000 +ARG AGENT_WORKDIR=/home/${user}/agent + +RUN mkdir -p /scripts + +COPY start_agent.sh /scripts/ + +# install python dependencies for the scripts +# ADD will checks that contentent of a file has changed. +ADD "https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/phabtalk/requirements.txt" requirements.txt +RUN pip3 install -r requirements.txt + +RUN groupadd -g ${gid} ${group} ;\ + useradd -c "buildkite user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} ;\ + mkdir /home/${user}/ccache + +RUN chown -R ${user}:${user} /home/${user} + +WORKDIR /home/${user} +ENV CCACHE_PATH=/mnt/disks/ssd0/ccache + +# configure locale +RUN sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\ + locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +# buildkite +RUN apt-get install -y apt-transport-https gnupg;\ + sh -c 'echo deb https://apt.buildkite.com/buildkite-agent stable main > /etc/apt/sources.list.d/buildkite-agent.list' ;\ + apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198 ;\ + apt-get update ;\ + apt-get install -y buildkite-agent + +VOLUME /credentials + +CMD ["/scripts/start_agent.sh"] + diff --git a/containers/agent-debian-testing-clang8-buildkite/start_agent.sh b/containers/agent-debian-testing-buildkite/start_agent.sh similarity index 100% rename from containers/agent-debian-testing-clang8-buildkite/start_agent.sh rename to containers/agent-debian-testing-buildkite/start_agent.sh diff --git a/containers/agent-debian-testing-clang8-ssd/Dockerfile b/containers/agent-debian-testing-ssd/Dockerfile similarity index 79% rename from containers/agent-debian-testing-clang8-ssd/Dockerfile rename to containers/agent-debian-testing-ssd/Dockerfile index 4baab8d..ac722a4 100644 --- a/containers/agent-debian-testing-clang8-ssd/Dockerfile +++ b/containers/agent-debian-testing-ssd/Dockerfile @@ -1,53 +1,56 @@ -FROM debian:testing - -RUN apt-get update ;\ - apt-get install -y --no-install-recommends locales \ - cmake ninja-build git ca-certificates clang lld ccache python3 build-essential \ - clang-tidy clang-format \ - python3-psutil arcanist zip wget \ - openjdk-11-jdk \ - python3-pip python3-setuptools \ - swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq; \ - apt-get clean - -# Make python3 default (needed by git-clang-format and others). -RUN rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python -# required for openssh server -RUN mkdir -p /run/sshd - -ARG user=jenkins -ARG group=jenkins -ARG uid=1000 -ARG gid=1000 -ARG AGENT_WORKDIR=/home/${user}/agent - -RUN mkdir -p /scripts - -# install the swarm client -RUN cd /scripts ;\ - wget https://repo.jenkins-ci.org/releases/org/jenkins-ci/plugins/swarm-client/3.17/swarm-client-3.17.jar ;\ - mv swarm-client-3.17.jar swarm-client.jar - -COPY start_agent.sh report_results.sh /scripts/ - -# install python dependencies for the scripts -RUN pip3 install -r https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/phabtalk/requirements.txt - -RUN groupadd -g ${gid} ${group} ;\ - useradd -c "Jenkins user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} ;\ - mkdir /home/${user}/ccache - -RUN chown -R ${user}:${user} /home/${user} - -WORKDIR /home/${user} -ENV CCACHE_PATH=/mnt/disks/ssd0/ccache - -# configure locale -RUN sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\ - locale-gen -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 - -CMD ["/scripts/start_agent.sh"] - +FROM debian:testing + +RUN echo "deb [trusted=yes] http://apt.llvm.org/buster/ llvm-toolchain-buster-10 main\n$(cat /etc/apt/sources.list)" > /etc/apt/sources.list;\ + apt-get update ;\ + apt-get install -y --no-install-recommends locales \ + cmake ninja-build git ca-certificates clang lld ccache python3 build-essential \ + clang-tidy clang-format \ + python3-psutil arcanist zip wget \ + openjdk-11-jdk \ + python3-pip python3-setuptools \ + swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq; \ + apt-get clean + +# Make python3 default (needed by git-clang-format and others). +RUN rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python +# required for openssh server +RUN mkdir -p /run/sshd + +ARG user=jenkins +ARG group=jenkins +ARG uid=1000 +ARG gid=1000 +ARG AGENT_WORKDIR=/home/${user}/agent + +RUN mkdir -p /scripts + +# install the swarm client +RUN cd /scripts ;\ + wget https://repo.jenkins-ci.org/releases/org/jenkins-ci/plugins/swarm-client/3.17/swarm-client-3.17.jar ;\ + mv swarm-client-3.17.jar swarm-client.jar + +COPY start_agent.sh report_results.sh /scripts/ + +# install python dependencies for the scripts +# ADD will checks that contentent of a file has changed. +ADD "https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/phabtalk/requirements.txt" requirements.txt +RUN pip3 install -r requirements.txt + +RUN groupadd -g ${gid} ${group} ;\ + useradd -c "Jenkins user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} ;\ + mkdir /home/${user}/ccache + +RUN chown -R ${user}:${user} /home/${user} + +WORKDIR /home/${user} +ENV CCACHE_PATH=/mnt/disks/ssd0/ccache + +# configure locale +RUN sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\ + locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +CMD ["/scripts/start_agent.sh"] + diff --git a/containers/agent-debian-testing-clang8-ssd/report_results.sh b/containers/agent-debian-testing-ssd/report_results.sh similarity index 100% rename from containers/agent-debian-testing-clang8-ssd/report_results.sh rename to containers/agent-debian-testing-ssd/report_results.sh diff --git a/containers/agent-debian-testing-clang8-ssd/start_agent.sh b/containers/agent-debian-testing-ssd/start_agent.sh similarity index 100% rename from containers/agent-debian-testing-clang8-ssd/start_agent.sh rename to containers/agent-debian-testing-ssd/start_agent.sh diff --git a/containers/build_deploy.sh b/containers/build_deploy.sh index 299a2b7..afc84f4 100755 --- a/containers/build_deploy.sh +++ b/containers/build_deploy.sh @@ -25,6 +25,6 @@ IMAGE_NAME="${1%/}" QUALIFIED_NAME="${GCR_HOSTNAME}/${GCP_PROJECT}/${IMAGE_NAME}" cd "${DIR}/${IMAGE_NAME}" -docker build -t ${IMAGE_NAME} . +docker build --no-cache -t ${IMAGE_NAME} . docker tag ${IMAGE_NAME} ${QUALIFIED_NAME} -docker push ${QUALIFIED_NAME} \ No newline at end of file +docker push ${QUALIFIED_NAME} \ No newline at end of file diff --git a/containers/build_run.sh b/containers/build_run.sh index 36e8f9b..4ae154f 100755 --- a/containers/build_run.sh +++ b/containers/build_run.sh @@ -14,7 +14,7 @@ # limitations under the License. # Starts a new instances of a docker image. Example: -# sudo build_run.sh agent-debian-testing-clang8-ssd /bin/bash +# sudo build_run.sh agent-debian-testing-ssd /bin/bash set -eux DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" diff --git a/docs/clang_tidy.md b/docs/clang_tidy.md new file mode 100644 index 0000000..3efce52 --- /dev/null +++ b/docs/clang_tidy.md @@ -0,0 +1,29 @@ +# clang-tidy checks +## Warning is not useful +If you found that a warning produced by clang-tidy is not useful: + +- If clang-tidy must not run for some files at all (e.g. lit test), please +[add files to blacklist](../scripts/clang-tidy.ignore). + +- Consider fixing or [suppressing diagnostic](https://clang.llvm.org/extra/clang-tidy/#suppressing-undesired-diagnostics) + if there is a good reason. + +- [File a bug](https://github.com/google/llvm-premerge-checks/issues/new?assignees=&labels=bug&template=bug_report.md&title=) + if build process should be improved. + +- If you believe that you found a clang-tidy bug then please keep in mind that clang-tidy version run by bot + might not be most recent. Please reproduce your issue on current version before submitting a bug to clang-tidy. + +## Review comments + +Build bot leaves inline comments only for a small subset of files that are not blacklisted for analysis (see above) *and* +specifically whitelisted for comments. + +That is done to avoid potential noise when a file already contains a number of warnings. + +If your are confident that some files are in good shape already, please +[whitelist them](../scripts/clang-tidy-comments.ignore). + +---- + +[about pre-merge checks](docs/user_doc.md) \ No newline at end of file diff --git a/docs/playbooks.md b/docs/playbooks.md index 94d0c88..bdfb9e9 100644 --- a/docs/playbooks.md +++ b/docs/playbooks.md @@ -6,11 +6,14 @@ General remarks: * GCP does not route any traffic to your services unless the service is "healthy". It might take a few minutes after startup before the services is classified as healthy. Until then you will only see some generic error message. -These are the steps to set up the build server on a clean infrastructure: +These are the steps to set up the build server on a clean infrastructure: 1. Configure the tools on your local machine: ```bash ./local_setup.sh ``` + If you not running docker under your user, you might need to + `sudo gcloud auth login --no-launch-browser && gcloud auth configure-docker` + before running other commands under sudo. 1. Delete the old cluster, if it still exists: ```bash cd kubernetes/cluster @@ -120,13 +123,21 @@ To spawn a new windows agent: 1. In the RDP session: run these commands in the CMD window to start the docker container: ```cmd powershell -Invoke-WebRequest -uri 'https://raw.githubusercontent.com/google/llvm-premerge-checks/master/kubernetes/windows_agent_bootstrap.ps1' -OutFile windows_agent_bootstrap.ps1 +Invoke-WebRequest -uri 'https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/windows_agent_bootstrap.ps1' -OutFile windows_agent_bootstrap.ps1 +.\windows_agent_bootstrap.ps1 +``` +1. Wait for the machine to reboot, then login again and store the `gsutil` credentials in `build-agent-results_key`. +TODO: add documentation on how to create these. +1. run this script to start containers: +```cmd +powershell +Invoke-WebRequest -uri 'https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/windows_agent_start.ps1' -OutFile windows_agent_bootstrap.ps1 .\windows_agent_bootstrap.ps1 ``` ## Testing scripts locally -Build and run agent docker image `sudo ./containers/build_run.sh agent-debian-testing-clang8-ssd /bin/bash`. +Build and run agent docker image `sudo ./containers/build_run.sh agent-debian-testing-ssd /bin/bash`. Within a container set environment variables similar to [pipeline](https://github.com/google/llvm-premerge-checks/blob/master/Jenkins/Phabricator-pipeline/Jenkinsfile). diff --git a/docs/user_doc.md b/docs/user_doc.md index 75adc63..041c399 100644 --- a/docs/user_doc.md +++ b/docs/user_doc.md @@ -44,17 +44,19 @@ Only then can the build server apply the patch locally and run the builds and te Once you're signed up, Phabricator will automatically trigger a build for every new patch you upload or every existing patch you modify. Phabricator shows the build results at the top of the entry: ![build status](images/diff_detail.png) +Bot will compile and run tests, run clang-format and [clang-tidy](docs/clang_tidy.md) on lines changed. + If a unit test failed, this is shown below the build status. You can also expand the unit test to see the details: ![unit test results](images/unit_tests.png) -After every build the build server will comment on your latest patch, so that you can also see the results for previous changes. The comment also contains a link to the log files: +After every build the build server will comment on your latest patch, so that you can also see the results for previous changes. +The comment also contains a link to the log files: ![bot comment](images/bot_comment.png) The build logs are stored for 90 days and automatically deleted after that. You can also trigger a build manually by using the "Run Plan Manually" link on the [Harbormaster page](https://reviews.llvm.org/harbormaster/plan/3/) and entering a revision ID in the pop-up window. - # Reporting issues If you notice any bugs, please create a [new issue](https://github.com/google/llvm-premerge-checks/issues/new). diff --git a/kubernetes/jenkins.yaml b/kubernetes/jenkins.yaml index 65f62cd..ba2b9be 100644 --- a/kubernetes/jenkins.yaml +++ b/kubernetes/jenkins.yaml @@ -151,18 +151,18 @@ spec: apiVersion: extensions/v1beta1 kind: Deployment metadata: - name: agent-debian-testing-clang8-ssd + name: agent-debian-testing-ssd namespace: jenkins spec: replicas: 2 template: metadata: labels: - app: agent-debian-testing-clang8-ssd + app: agent-debian-testing-ssd spec: containers: - name: agent-debian-testing-clang8 - image: gcr.io/llvm-premerge-checks/agent-debian-testing-clang8-ssd + image: gcr.io/llvm-premerge-checks/agent-debian-testing-ssd ports: - containerPort: 22 resources: diff --git a/scripts/clang-format.ignore b/scripts/clang-format.ignore new file mode 100644 index 0000000..c43ec0c --- /dev/null +++ b/scripts/clang-format.ignore @@ -0,0 +1 @@ +# Patterns for clang-format to ignore. \ No newline at end of file diff --git a/scripts/clang-tidy-comments.ignore b/scripts/clang-tidy-comments.ignore new file mode 100644 index 0000000..6663111 --- /dev/null +++ b/scripts/clang-tidy-comments.ignore @@ -0,0 +1,4 @@ +# Files that are allowed by clang-tidy.ignore but should not receive inline review comments. +# Right now it works in whitelist mode and only some files / directories are whitelisted. +* +!clang-tools-extra/clangd/** \ No newline at end of file diff --git a/scripts/clang-tidy.ignore b/scripts/clang-tidy.ignore new file mode 100644 index 0000000..4d74be8 --- /dev/null +++ b/scripts/clang-tidy.ignore @@ -0,0 +1,44 @@ +# Files to be ignored by clang-tidy. Will not appear in short report and inline comments. +libcxxabi/test +libcxxabi/test/libcxxabi/test +libcxx/test +libcxx/utils/libcxx/test +libcxx/utils/google-benchmark/test +llvm/test +llvm/utils/gn/secondary/llvm/test +llvm/utils/gn/secondary/lld/test +llvm/utils/gn/secondary/clang-tools-extra/test +llvm/utils/gn/secondary/clang-tools-extra/clangd/test +llvm/utils/gn/secondary/compiler-rt/test +llvm/utils/gn/secondary/clang/test +llvm/utils/benchmark/test +polly/test +lldb/examples/test +lldb/test +lldb/utils/test +lldb/tools/intel-features/intel-mpx/test +lldb/packages/Python/lldbsuite/test +lldb/packages/Python/lldbsuite/test/tools/lldb-server/test +lldb/packages/Python/lldbsuite/test/commands/expression/test +lldb/packages/Python/lldbsuite/test/test_runner/test +lldb/third_party/Python/module/unittest2/unittest2/test +lld/test +clang-tools-extra/test +clang-tools-extra/clangd/clients/clangd-vscode/test +clang-tools-extra/clangd/test +pstl/test +libc/test +llgo/test +compiler-rt/test +compiler-rt/test/builtins/Unit/ppc/test +compiler-rt/test/builtins/Unit/test +debuginfo-tests/dexter/dex/tools/test +debuginfo-tests/dexter/feature_tests/subtools/test +clang/test +libclc/test +mlir/test +openmp/libomptarget/test +openmp/libomptarget/deviceRTLs/nvptx/test +openmp/runtime/test +libunwind/test +libunwind/test/libunwind/test diff --git a/scripts/ignore_diff.py b/scripts/ignore_diff.py new file mode 100755 index 0000000..a376948 --- /dev/null +++ b/scripts/ignore_diff.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# Copyright 2020 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import sys +import pathspec + +# Takes an output of git diff and removes files ignored by patten specified by ignore file. +def main(): + argv = sys.argv[1:] + if not argv: + print("Please provide a path to .ignore file.") + sys.exit(1) + ignore = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, + open(argv[0], 'r').readlines()) + good = True + for line in sys.stdin: + match = re.search(r'^diff --git a/(.*) b/(.*)$', line) + if match: + good = not (ignore.match_file(match.group(1)) and ignore.match_file(match.group(2))) + if not good: + continue + sys.stdout.write(line) + + +if __name__ == "__main__": + main() diff --git a/scripts/lint.sh b/scripts/lint.sh index 7618d4a..e54f1d8 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -21,16 +21,17 @@ set -eux echo "Running linters... =====================================" cd "${WORKSPACE}" + +# clang-format # Let clang format apply patches --diff doesn't produces results in the format we want. git-clang-format set +e -git diff -U0 --exit-code > "${TARGET_DIR}"/clang-format.patch -STATUS="${PIPESTATUS[0]}" +git diff -U0 --exit-code | "${SCRIPT_DIR}/ignore_diff.py" "${SCRIPT_DIR}/clang-format.ignore" > "${TARGET_DIR}"/clang-format.patch set -e # Revert changes of git-clang-format. git checkout -- . -# TODO: clang tidy is currently disabled, see https://github.com/google/llvm-premerge-checks/issues/91 -# git diff HEAD^ | clang-tidy-diff -p1 -quiet > "${TARGET_DIR}"/clang-tidy.txt +# clang-tidy +git diff -U0 HEAD | "${SCRIPT_DIR}/ignore_diff.py" "${SCRIPT_DIR}/clang-tidy.ignore" | clang-tidy-diff -p1 -quiet > "${TARGET_DIR}"/clang-tidy.txt echo "linters completed ======================================" diff --git a/scripts/phabtalk/apply_patch.py b/scripts/phabtalk/apply_patch.py index 2a9d7c0..ef1f24b 100755 --- a/scripts/phabtalk/apply_patch.py +++ b/scripts/phabtalk/apply_patch.py @@ -1,4 +1,4 @@ -#!/bin/env python3 +#!/usr/bin/env python3 # Copyright 2019 Google LLC # # Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); @@ -87,6 +87,8 @@ class ApplyPatch: 'master branch instead...'.format(self.git_hash)] subprocess.check_call('git checkout master', stdout=sys.stdout, stderr=sys.stderr, shell=True) + subprocess.check_call('git show -s', stdout=sys.stdout, + stderr=sys.stderr, shell=True) print('git checkout completed.') def _apply_patch(self): diff --git a/scripts/phabtalk/apply_patch2.py b/scripts/phabtalk/apply_patch2.py index d436b2e..211f596 100755 --- a/scripts/phabtalk/apply_patch2.py +++ b/scripts/phabtalk/apply_patch2.py @@ -21,7 +21,7 @@ import sys from typing import List, Optional, Tuple from phabricator import Phabricator -from git import Repo +from git import Repo, GitCommandError class ApplyPatch: """Apply a diff from Phabricator on local working copy. @@ -76,9 +76,14 @@ class ApplyPatch: try: revision_id, dependencies, base_revision = self._get_dependencies() print('Checking out {}...'.format(base_revision)) - self.repo.git.checkout(base_revision) + try: + self.repo.git.checkout(base_revision) + except GitCommandError: + print('ERROR checking out revision {}. It`s not in the ' + 'repository. Using master instead.'.format(base_revision)) + self.repo.git.checkout('master') print('Revision is {}'.format(self.repo.head.commit.hexsha)) - print('Cleanup...') + print('git reset, git cleanup...') self.repo.git.reset('--hard') self.repo.git.clean('-fdx') print('Analyzing {}'.format(diff_to_str(revision_id))) @@ -139,7 +144,7 @@ class ApplyPatch: """Download and apply a diff to the local working copy.""" print('Applying diff {} for revision {}...'.format(diff_id, diff_to_str(revision_id))) diff = self.phab.differential.getrawdiff(diffID=diff_id).response - proc = subprocess.run('git apply --whitespace=nowarn --binary -v', input=diff, shell=True, text=True, + proc = subprocess.run('patch -p1', input=diff, shell=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if proc.returncode != 0: raise Exception('Applying patch failed:\n{}'.format(proc.stdout + proc.stderr)) diff --git a/scripts/phabtalk/phabtalk.py b/scripts/phabtalk/phabtalk.py index a6609df..8768004 100755 --- a/scripts/phabtalk/phabtalk.py +++ b/scripts/phabtalk/phabtalk.py @@ -22,8 +22,10 @@ import os import re import socket import time +import urllib from typing import Optional +import pathspec from lxml import etree from phabricator import Phabricator @@ -48,6 +50,7 @@ class BuildReport: self.lint[key] = [] self.lint[key].append(m) + class PhabTalk: """Talk to Phabricator to upload build results. See https://secure.phabricator.com/conduit/method/harbormaster.sendmessage/ @@ -63,7 +66,7 @@ class PhabTalk: def dryrun(self): return self._phab is None - def _get_revision_id(self, diff: str): + def get_revision_id(self, diff: str): """Get the revision ID for a diff from Phabricator.""" if self.dryrun: return None @@ -75,7 +78,7 @@ class PhabTalk: """Add a comment to a differential based on the diff_id""" print('Sending comment to diff {}:'.format(diff_id)) print(text) - self._comment_on_revision(self._get_revision_id(diff_id), text) + self._comment_on_revision(self.get_revision_id(diff_id), text) def _comment_on_revision(self, revision: str, text: str): """Add comment on a differential based on the revision id.""" @@ -213,18 +216,26 @@ def _add_clang_format(report: BuildReport, results_dir: str, report.success = success and report.success -def _add_clang_tidy(report: BuildReport, results_dir: str, results_url: str, workspace: str, clang_tidy_file: str): +def _add_clang_tidy(report: BuildReport, results_dir: str, results_url: str, workspace: str, clang_tidy_file: str, + clang_tidy_ignore: str): # Typical message looks like # [..]/clang/include/clang/AST/DeclCXX.h:3058:20: error: no member named 'LifetimeExtendedTemporary' in 'clang::Decl' [clang-diagnostic-error] - pattern = '^{}/([^:]*):(\\d+):(\\d+): (.*): (.*)'.format(workspace) - success = True + errors_count = 0 + warn_count = 0 + inline_comments = 0 present = (clang_tidy_file is not None) and os.path.exists(os.path.join(results_dir, clang_tidy_file)) if not present: print('clang-tidy result {} is not found'.format(clang_tidy_file)) report.comments.append(section_title('clang-tidy', False, False)) return - + present = (clang_tidy_ignore is not None) and os.path.exists(clang_tidy_ignore) + if not present: + print('clang-tidy ignore file {} is not found'.format(clang_tidy_ignore)) + report.comments.append(section_title('clang-tidy', False, False)) + return + ignore = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, + open(clang_tidy_ignore, 'r').readlines()) p = os.path.join(results_dir, clang_tidy_file) for line in open(p, 'r'): match = re.search(pattern, line) @@ -234,21 +245,34 @@ def _add_clang_tidy(report: BuildReport, results_dir: str, results_url: str, wor char_pos = match.group(3) severity = match.group(4) text = match.group(5) + text += '\n[[https://github.com/google/llvm-premerge-checks/blob/master/docs/clang_tidy.md#warning-is-not' \ + '-useful | not useful]] ' if severity in ['warning', 'error']: - success = False - report.addLint({ - 'name': 'clang-tidy', - 'severity': 'warning', - 'code': 'clang-tidy', - 'path': file_name, - 'line': int(line_pos), - 'char': int(char_pos), - 'description': '{}: {}'.format(severity, text), - }) - + if severity == 'warning': + warn_count += 1 + if severity == 'error': + errors_count += 1 + if ignore.match_file(file_name): + print('{} is ignored by pattern and no comment will be added'.format(file_name)) + else: + inline_comments += 1 + report.addLint({ + 'name': 'clang-tidy', + 'severity': 'warning', + 'code': 'clang-tidy', + 'path': file_name, + 'line': int(line_pos), + 'char': int(char_pos), + 'description': '{}: {}'.format(severity, text), + }) + success = errors_count + warn_count == 0 comment = section_title('clang-tidy', success, present) if not success: - comment += 'Please fix [[ {}/{} | clang-tidy findings ]].'.format(results_url, clang_tidy_file) + comment += "clang-tidy found [[ {}/{} | {} errors and {} warnings]]. {} of them are added as review comments " \ + "below ([[https://github.com/google/llvm-premerge-checks/blob/master/docs/clang_tidy.md#review" \ + "-comments | why?]])." \ + .format(results_url, clang_tidy_file, errors_count, warn_count, inline_comments) + report.comments.append(comment) report.success = success and report.success @@ -355,10 +379,20 @@ def main(): report.success = False _add_test_results(report, args.results_dir, args.test_result_file) - _add_clang_tidy(report, args.results_dir, args.results_url, args.workspace, args.clang_tidy_result) + _add_clang_tidy(report, args.results_dir, args.results_url, args.workspace, args.clang_tidy_result, + args.clang_tidy_ignore) _add_clang_format(report, args.results_dir, args.results_url, args.clang_format_patch) _add_links_to_artifacts(report, args.results_dir, args.results_url) + p = PhabTalk(args.conduit_token, args.host, args.dryrun) + title = 'Issue with build for {} ({})'.format(p.get_revision_id(args.diff_id), args.diff_id) + report.comments.append( + '//Pre-merge checks is in beta. [[ https://github.com/google/llvm-premerge-checks/issues/new?assignees' + '=&labels=bug&template=bug_report.md&title={} | Report issue]]. ' + 'Please [[ https://reviews.llvm.org/project/update/78/join/ | join beta ]] or ' + '[[ https://github.com/google/llvm-premerge-checks/issues/new?assignees=&labels=enhancement&template' + '=&title=enable%20checks%20for%20{{PATH}} | enable it for your project ]].//'.format( + urllib.parse.quote(title))) p.submit_report(args.diff_id, args.ph_id, report, args.buildresult) @@ -380,6 +414,9 @@ def _parse_args(): parser.add_argument('--clang-tidy-result', type=str, default=None, dest='clang_tidy_result', help="path to diff produced by git-clang-tidy, relative to results-dir") + parser.add_argument('--clang-tidy-ignore', type=str, default=None, + dest='clang_tidy_ignore', + help="path to file with patters to exclude commenting on for clang-tidy findings") parser.add_argument('--results-dir', type=str, default=None, required=True, dest='results_dir', help="directory of all build artifacts") diff --git a/scripts/phabtalk/requirements.txt b/scripts/phabtalk/requirements.txt index 3c9d93e..4b20cc1 100644 --- a/scripts/phabtalk/requirements.txt +++ b/scripts/phabtalk/requirements.txt @@ -1,4 +1,5 @@ -phabricator==0.7.0 -lxml==4.4.1 -gitpython==3.0.5 -retrying==1.3.3 \ No newline at end of file +phabricator==0.7.0 +lxml==4.4.1 +gitpython==3.0.5 +retrying==1.3.3 +pathspec==0.7.0 \ No newline at end of file diff --git a/scripts/run_cmake.sh b/scripts/run_cmake.sh index 295c562..f40e02a 100755 --- a/scripts/run_cmake.sh +++ b/scripts/run_cmake.sh @@ -33,6 +33,7 @@ cmake -GNinja ../llvm -DCMAKE_BUILD_TYPE=Release -D LLVM_ENABLE_LLD=ON \ RETURN_CODE="${PIPESTATUS[0]}" set -e +rm -f "$WORKSPACE/compile_commands.json" ln -s "$WORKSPACE"/build/compile_commands.json "$WORKSPACE" cp CMakeCache.txt ${TARGET_DIR} diff --git a/scripts/windows_agent_bootstrap.ps1 b/scripts/windows_agent_bootstrap.ps1 index 276f3a5..7303644 100644 --- a/scripts/windows_agent_bootstrap.ps1 +++ b/scripts/windows_agent_bootstrap.ps1 @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +# 1st stage of the installation process. +# This script only needs to be run once per machine. + Write-Host "Initializing local SSD..." New-Variable -Name diskid -Value (Get-Disk -FriendlyName "Google EphemeralDisk").Number # TODO: check if machine has an SSD -# TODO: only do this, if SSD is not yet usable +# TODO: only do this, if SSD is not yet partioned and formatted Initialize-Disk -Number $diskid New-Partition -DiskNumber $diskid -UseMaximumSize -AssignDriveLetter Format-Volume -DriveLetter D @@ -23,11 +26,21 @@ Format-Volume -DriveLetter D Write-Host "install chocolately as package manager..." iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')) choco feature disable --name showDownloadProgress -choco install git +choco install -y git -Write-Host "Mounting result storage..." -Install-WindowsFeature NFS-Client -net use E: \\results.local\exports /PERSISTENT:YES +# install Docker +Install-PackageProvider -Name NuGet -Force +Install-Module -Name DockerMsftProvider -Repository PSGallery -Force +Install-Package -Name docker -ProviderName DockerMsftProvider -Force +Set-Service -Name docker -StartupType Automatic -Set-Location D:\ -git clone https://github.com/google/llvm-premerge-checks +# install gcloud and authenticate access to gcr.io registry +# TODO: find a better way to install the Google Cloud SDK, avoid ingoring the checksum +choco install -y gcloudsdk --ignore-checksums +gcloud auth docker + +# move docker folder to SSD to get better IO performance +cmd /C "mklink /j C:\ProgramData\Docker D:\docker" + +# Reboot +Restart-Computer -Force diff --git a/scripts/windows_agent_install.ps1 b/scripts/windows_agent_install.ps1 deleted file mode 100644 index 46c3b40..0000000 --- a/scripts/windows_agent_install.ps1 +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2019 Google LLC - -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# https://llvm.org/LICENSE.txt - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -. ${PSScriptRoot}\common.ps1 - -Write-Host "Installing Visual Studio build tools..." -choco install -y visualcpp-build-tools --version 15.0.26228.20170424 -y --params "'/IncludeOptional'" -Write-Host 'Waiting for Visual C++ Build Tools to finish' -Wait-Process -Name vs_installer - -Write-Host "Installing misc tools" -# install other tools as described in https://llvm.org/docs/GettingStartedVS.html -# and a few more that were not documented... -choco install -y git python2 ninja gnuwin cmake -pip install psutil - -Write-Host "Setting environment variables..." -[System.Environment]::SetEnvironmentVariable('PYTHONIOENCODING', 'UTF-8', [System.EnvironmentVariableTarget]::User) -$oldpath=[System.Environment]::GetEnvironmentVariable('path',[System.EnvironmentVariableTarget]::User) -[System.Environment]::SetEnvironmentVariable('path', $oldpath + 'c:\Program Files (x86)\GnuWin32\bin;C:\Program Files\CMake\bin', [System.EnvironmentVariableTarget]::User) -# support long file names during git checkout -Write-Host "Setting git config..." -git config --system core.longpaths true -git config --global core.autocrlf false - -# Above: genric LLVM-related things -#------------- -# Below: Jenkins specific things - -Write-Host "Installing openjdk..." -choco install -y openjdk - -Write-Host "Installing Jenkins swarm agent..." -$SWARM_PLUGIN_URL="https://repo.jenkins-ci.org/releases/org/jenkins-ci/plugins/swarm-client/3.17/swarm-client-3.17.jar" -$SWARM_PLUGIN_JAR="C:\jenkins\swarm-client.jar" -mkdir c:\jenkins -Invoke-WebRequest -Uri $SWARM_PLUGIN_URL -OutFile $SWARM_PLUGIN_JAR - -# TODO: put build-agent-results_key.json on the machine somehow -gcloud auth activate-service-account --key-file C:\jenkins\build-agent-results_key.json \ No newline at end of file diff --git a/scripts/windows_agent_start.ps1 b/scripts/windows_agent_start.ps1 index 3a218d9..95de34c 100644 --- a/scripts/windows_agent_start.ps1 +++ b/scripts/windows_agent_start.ps1 @@ -1,32 +1,30 @@ # Copyright 2019 Google LLC -# + # Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# + # https://llvm.org/LICENSE.txt -# + # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# Pull and start the Docker container for a Windows agent. +# To setup a Windows agent see docs/playbooks.md -. ${PSScriptRoot}\common.ps1 +$NAME="agent-windows-jenkins" +$IMAGE="gcr.io/llvm-premerge-checks/${NAME}" -$JENKINS_SERVER="jenkins.local" -$AGENT_ROOT="D:\" -$SWARM_PLUGIN_JAR="C:\jenkins\swarm-client.jar" - -# move temp dir to local SSD -mkdir D:\temp -$env:temp="D:\temp" -$env:tmp=$env:temp - -java -jar ${SWARM_PLUGIN_JAR} ` - -master http://${JENKINS_SERVER}:8080 ` - -executors 1 ` - -fsroot ${AGENT_ROOT} ` - -labels windows ` - -name $env:computername \ No newline at end of file +docker pull ${IMAGE} +docker stop ${NAME} +docker rm ${NAME} +docker run ` + -v D:\:C:\ws ` + -v C:\credentials:C:\credentials ` + -e PARENT_HOSTNAME=$env:computername ` + --restart unless-stopped ` + --name ${NAME} ` + ${IMAGE} \ No newline at end of file