Upgrade Windows agents to Visual Studio 2019 and sccache (#162)

* added cmake parameters for sccache

* added sccache

* removed stray debug output

* starting sccache with VS environment

* added container for Visual Studio 2019

* fixed comments

* considering WIndows version

* updated to vs2019

* using MS install method

* snapshot of VS2019 experiments

* using --installRecommended

* cleanup of dockerfile

* updated path

* fixed dockerfile

* dumped version number

* exclude for virus scan

* added testing option

this does not start the agent

* write results on failures

* added timeouts for pipelines

* moving master pipelines to python scripts

* added flang to automatic project selection

based on #159, this will enable flang for beta testers

* added persistent workspace for testing containers

* added secure delete function

* added better log message

* deleting read-only files

* checking existence before setting flags

* using unlink

* deleting recursively

* using pathlib for chmod

* using custom workspace

* fixed drive

* separate handling of single files

* simplified read-only handling again

* removed vsdevcmd calls

as it's already set in the docker Entrypoint

* renamed container folders

* windows version not needed any more

* bumped version number

* bumped version number

* script cleanup

* added cmake parameters for sccache

* added sccache

* starting sccache with VS environment

* added container for Visual Studio 2019

* fixed comments

* considering WIndows version

* updated to vs2019

* using MS install method

* snapshot of VS2019 experiments

* using --installRecommended

* cleanup of dockerfile

* updated path

* fixed dockerfile

* dumped version number

* removed vsdevcmd calls

as it's already set in the docker Entrypoint

* renamed container folders

* windows version not needed any more

* bumped version number

* bumped version number

* script cleanup

* removed sccache from vs2017

* making windows image configurable

* added versioning

* created windows BETA pipeline

* added Jenkins label for vs2019

Jenkins/BETA-Phabricator-windows-pipeline/Jenkinsfile

@@ -0,0 +1,169 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://llvm.org/LICENSE.txt
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+def success = true
+
+pipeline {
+    agent { label 'windows' }
+    parameters {
+        string(name: 'DIFF_ID')
+        string(name: 'PHID')
+        string(name: 'REV_ID')
+    }
+    environment {
+        CONDUIT_TOKEN = credentials('phabricator-conduit-token')
+        PHABRICATOR_HOST = 'https://reviews.llvm.org'
+        PHAB_LOG = "${WORKSPACE}/build/.phabricator-comment"
+        MY_BUILD_ID = "${JOB_BASE_NAME}-${BUILD_NUMBER}"
+        RESULT_URL = "https://storage.cloud.google.com/llvm-premerge-checks/results/${MY_BUILD_ID}"
+        SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts"
+        // store all build results here, will be uploaded to GCS later
+        RESULT_DIR = "${WORKSPACE}\\results"
+        LLVM_DIR = "${WORKSPACE}\\llvm-project"
+    }
+    options {
+        timeout(time:2, unit:'HOURS')
+    }
+    stages {
+        stage("build info"){
+            steps {
+                echo "Building diff ${DIFF_ID} with PHID ${PHID} for Revision ${REV_ID}"
+                script {
+                    currentBuild.displayName += " D${REV_ID}"
+                    currentBuild.description = "<a href='https://reviews.llvm.org/D${REV_ID}'>D${REV_ID}</a>"
+                }
+                script {
+                    success = true
+                    failure_message = ""
+                }
+            }
+        }
+        stage("git checkout"){
+            steps {
+                echo "getting llvm-premerge-checks... "
+                dir("llvm-premerge-checks")
+                {
+                    git url: 'https://github.com/google/llvm-premerge-checks.git'
+                }
+                echo "getting llvm-project... "
+                dir("llvm-project")
+                {
+                    git url: 'https://github.com/llvm-premerge-tests/llvm-project.git', branch: "phab-diff-${DIFF_ID}"
+                    powershell 'git clean -fdx'
+                    powershell 'git show -s'
+                }
+                // create ${RESULT_DIR}
+                powershell """
+                    Remove-Item ${RESULT_DIR} -Recurse -ErrorAction Ignore
+                    New-Item -ItemType Directory -Force -Path ${RESULT_DIR} | Out-Null
+                    """
+            }            
+        }
+        stage('CMake') {
+            steps {
+                dir("${LLVM_DIR}"){
+                    powershell "${SCRIPT_DIR}/run_cmake.py detect"
+                }
+            }
+            post {
+                failure {
+                    script {
+                        success = false
+                        failure_message = "Failed to run CMake"
+                    }
+                }
+            }
+        }
+        stage('ninja all') {
+            steps {
+                dir("${LLVM_DIR}"){
+                    powershell "${SCRIPT_DIR}/run_ninja.py all"
+                }
+            }
+            post {
+                failure {
+                    script {
+                        success = false
+                        failure_message = "Failed to run ninja all"
+                    }
+                }
+            }
+        }
+        stage('ninja check-all') {
+            steps {
+                dir("${LLVM_DIR}"){
+                    powershell "${SCRIPT_DIR}/run_ninja.py check-all"
+                }
+            }
+            post {
+                failure {
+                    script {
+                        success = false
+                        failure_message = "Failed to run ninja check-all"
+                    }
+                }
+            }
+        }
+    }
+    post { 
+        always { 
+            script {
+                if (success) {
+                    currentBuild.result = "SUCCESS"
+                } else {
+                    currentBuild.result = "FAILURE"
+                }
+            }
+            echo "Uploading logs to ${RESULT_URL} ..."
+            dir("${RESULT_DIR}") {
+                // gather all result files in a folder
+                powershell """
+                    \$ErrorActionPreference = 'Continue'
+                    Write-Host "Getting the console log..."
+                    Invoke-WebRequest -OutFile console-log.txt -uri "http://jenkins.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText" -ErrorAction "Continue"
+                    
+                    Write-Host "Copying build artefacts..."
+                    Copy-Item "${LLVM_DIR}\\choose_projects.log"
+                    Copy-Item "${LLVM_DIR}\\build\\CMakeCache.txt"
+                    Copy-Item "${LLVM_DIR}\\build\\test-results.xml"
+                """                               
+            }
+            /// send results to Phabricator
+            echo "Sending build feedback to Phabricator..."
+            bat """
+                python ${SCRIPT_DIR}/phabtalk/phabtalk.py "${PHID}" "${DIFF_ID}" ^
+                    --workspace "${LLVM_DIR}" ^
+                    --conduit-token "${CONDUIT_TOKEN}" ^
+                    --test-result-file "test-results.xml" ^
+                    --host "${PHABRICATOR_HOST}/api/" ^
+                    --results-dir "${RESULT_DIR}" ^
+                    --results-url "${RESULT_URL}" ^
+                    --failures "${failure_message}" ^
+                    --buildresult ${currentBuild.result} ^
+                    --name "windows"
+                """            
+            dir("${RESULT_DIR}") {
+                // upload results to
+                // Google Cloud Storage
+                powershell """
+                    Write-Host "Uploading results to GCS..."
+                    \$ErrorActionPreference = 'Continue'
+                    gsutil cp *.* gs://llvm-premerge-checks/results/${MY_BUILD_ID}/
+                    Write-Host "Done."
+                """
+            }
+            echo "Done."
+        } 
+    }
+}

containers/agent-windows-jenkins/Dockerfile

@@ -2,7 +2,7 @@
 
 # use windows server core image
 ARG agent_windows_version
-FROM gcr.io/llvm-premerge-checks/agent-windows:$agent_windows_version
+FROM gcr.io/llvm-premerge-checks/agent-windows-vs2019:${agent_windows_version}
 
 # install java
 RUN choco install -y openjdk
@@ -18,12 +18,9 @@ RUN powershell -NoProfile -InputFormat None -Command `
 RUN pip install gsutil
 VOLUME C:\credentials
 
-# install python dependencies for the scripts
-RUN pip install -r https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/requirements.txt
-
 # temporary directory, can be mounted on host if required 
 VOLUME C:\Temp
 
 # start swarm plugin
 COPY start_agent.ps1 c:\jenkins
-CMD powershell c:\jenkins\start_agent.ps1
+ENTRYPOINT ["C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass", "C:\\jenkins\\start_agent.ps1"]

containers/agent-windows-jenkins/start_agent.ps1

@@ -29,5 +29,5 @@ java -jar ${env:SWARM_PLUGIN_JAR} `
     -master http://${JENKINS_SERVER}:8080 `
     -executors 1 `
     -fsroot ${AGENT_ROOT} `
-    -labels windows `
+    -labels "windows vs2019" `
     -name ${env:PARENT_HOSTNAME}

containers/agent-windows-vs2019/Dockerfile

@@ -0,0 +1,66 @@
+# escape=`
+FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2019
+
+# Restore the default Windows shell for correct batch processing.
+SHELL ["cmd", "/S", "/C"]
+
+# Download the Build Tools bootstrapper.
+ADD https://aka.ms/vs/16/release/vs_buildtools.exe C:\TEMP\vs_buildtools.exe
+
+# Download channel for fixed install.
+ARG CHANNEL_URL=https://aka.ms/vs/16/release/channel
+ADD ${CHANNEL_URL} C:\TEMP\VisualStudio.chman
+
+# Install Build Tools with C++ workload.
+#   - Documentation for docker installation 
+#     https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2019
+#   - Documentation on workloads 
+#     https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools?view=vs-2019#c-build-tools
+#   - Documentation on flags
+#     https://docs.microsoft.com/en-us/visualstudio/install/use-command-line-parameters-to-install-visual-studio?view=vs-2019
+RUN C:\TEMP\vs_buildtools.exe --quiet --wait --norestart --nocache `
+    --channelUri C:\TEMP\VisualStudio.chman `
+    --installChannelUri C:\TEMP\VisualStudio.chman `
+    --installPath C:\BuildTools `
+    --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended `
+    || IF "%ERRORLEVEL%"=="3010" EXIT 0
+
+# install chocolately as package manager
+RUN powershell -NoProfile -InputFormat None -Command `
+    iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')) ; `
+    choco feature disable --name showDownloadProgress
+
+# install tools as described in https://llvm.org/docs/GettingStartedVS.html
+# and a few more that were not documented...
+RUN choco install -y ninja
+RUN choco install -y git
+RUN choco install -y cmake --version 3.15.4
+RUN choco install -y python3
+RUN choco install -y gnuwin
+RUN choco install -y sccache
+# set local cache folder for sccache
+ENV SCCACHE_DIR=C:\ws\sccache
+RUN pip install psutil
+
+# install python dependencies for the scripts
+RUN pip install -r https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/requirements.txt
+
+# configure Python encoding
+ENV PYTHONIOENCODING=UTF-8
+
+# update the path variable    
+RUN powershell -NoProfile -InputFormat None  -Command `
+    $path = $env:path + ';c:\Program Files (x86)\GnuWin32\bin;C:\Program Files\CMake\bin'; `
+    Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment\' -Name Path -Value $path
+
+# use this folder to store the worksapce'
+VOLUME C:\ws
+WORKDIR C:\ws
+
+# support long file names during git checkout
+RUN git config --system core.longpaths true & `
+    git config --global core.autocrlf false
+
+# Define the entry point for the docker container.
+# This entry point starts the developer command prompt and launches the PowerShell shell.
+ENTRYPOINT ["C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"]

containers/build_deploy.ps1

@@ -34,21 +34,15 @@ $QUALIFIED_NAME="${GCR_HOSTNAME}/${GCP_PROJECT}/${IMAGE_NAME}"
 Push-Location "$PSScriptRoot\$IMAGE_NAME"
 $container_version=[int](Get-Content $VERSION_FILE)
 $container_version+=1
-$agent_windows_version=Get-Content "../agent-windows/$VERSION_FILE"
+$agent_windows_version=Get-Content "../agent-windows-vs2019/$VERSION_FILE"
 
 Write-Host "Building ${IMAGE_NAME}:${container_version}..."
 Write-Host "Using windows-agent ${agent_windows_version}"
 
-# TODO: get current Windows version number from host via "cmd /c ver"
-# to solve these issues: https://stackoverflow.com/questions/43123851/unable-to-run-cygwin-in-windows-docker-container/52273489#52273489
-$windows_version="10.0.17763.1039"
-Write-Host "Using windows version ${windows_version}"
-
 Invoke-Call -ScriptBlock {
     docker build . `
         -t ${IMAGE_NAME}:${container_version} `
         -t ${IMAGE_NAME}:latest `
-        --build-arg windows_version=$windows_version `
         --build-arg agent_windows_version=$agent_windows_version
     }
 Invoke-Call -ScriptBlock {

containers/build_run.ps1

@@ -21,15 +21,25 @@ param(
 
 # set script to stop on first error
 $ErrorActionPreference = "Stop"
+$agent_windows_version=Get-Content "../agent-windows-vs2019/VERSION"
 
 # some docs recommend setting 2GB memory limit
- docker build --memory 2GB -t $IMAGE_NAME --build-arg token=$token "$PSScriptRoot\$IMAGE_NAME"
+docker build `
+    --memory 2GB `
+    -t $IMAGE_NAME `
+    --build-arg token=$token  `
+    --build-arg agent_windows_version=$agent_windows_version `
+    "$PSScriptRoot\$IMAGE_NAME"
 If ($LastExitCode -ne 0) {
     exit
 }
 
 # mount a persistent workspace for experiments
-docker run -it -v D:\:C:\ws -v C:\credentials:C:\credentials -e PARENT_HOSTNAME=$env:computername $IMAGE_NAME $CMD
+docker run -it `
+    -v D:\:C:\ws `
+    -v C:\credentials:C:\credentials `
+    -e PARENT_HOSTNAME=$env:computername `
+    $IMAGE_NAME $CMD
 If ($LastExitCode -ne 0) {
     exit
 }

scripts/run_cmake.py

@@ -101,8 +101,14 @@ def _create_args(config: Configuration, llvm_enable_projects: str) -> List[str]:
     arguments.extend(config.general_cmake_arguments)
     arguments.extend(config.specific_cmake_arguments)
 
+    # enable sccache
+    if 'SCCACHE_DIR' in os.environ:
+        arguments.extend([
+            '-DCMAKE_C_COMPILER_LAUNCHER=sccache',
+            '-DCMAKE_CXX_COMPILER_LAUNCHER=sccache',
+        ])
     # enable ccache if the path is set in the environment
-    if 'CCACHE_PATH' in os.environ:
+    elif 'CCACHE_PATH' in os.environ:
         arguments.extend([
             '-D LLVM_CCACHE_BUILD=ON',
             '-D LLVM_CCACHE_DIR={}'.format(os.environ['CCACHE_PATH']),
@@ -132,11 +138,6 @@ def run_cmake(projects: str, repo_path: str, config_file_path: str = None, *, dr
     arguments = _create_args(config, llvm_enable_projects)
     cmd = 'cmake ' + ' '.join(arguments)
     
-    # On Windows: configure Visutal Studio before running cmake
-    if config.operating_system == OperatingSystem.Windows:
-        # FIXME: move this path to a config file
-        cmd = r'"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64 && ' + cmd
-
     print('Running cmake with these arguments:\n{}'.format(cmd))
     if dryrun:
         print('Dryrun, not invoking CMake!')

scripts/run_ninja.py

@@ -21,12 +21,6 @@ import subprocess
 def run_ninja(target: str, repo_path: str):   
     build_dir = os.path.join(repo_path, 'build')
     cmd = 'ninja {}'.format(target)
-    
-    # On Windows: configure Visutal Studio before running ninja
-    if platform.system() == 'Windows':
-        # FIXME: move this path to a config file
-        cmd = r'"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64 && ' + cmd
-
     subprocess.check_call(cmd, shell=True, cwd=build_dir)