From a47751ebe8ed926ebcf467561d77b30955fe8eb6 Mon Sep 17 00:00:00 2001 From: ChristianKuehnel Date: Tue, 21 Apr 2020 11:46:41 +0200 Subject: [PATCH] Upgrade Windows agents to Visual Studio 2019 and sccache (#162) * added cmake parameters for sccache * added sccache * removed stray debug output * starting sccache with VS environment * added container for Visual Studio 2019 * fixed comments * considering WIndows version * updated to vs2019 * using MS install method * snapshot of VS2019 experiments * using --installRecommended * cleanup of dockerfile * updated path * fixed dockerfile * dumped version number * exclude for virus scan * added testing option this does not start the agent * write results on failures * added timeouts for pipelines * moving master pipelines to python scripts * added flang to automatic project selection based on #159, this will enable flang for beta testers * added persistent workspace for testing containers * added secure delete function * added better log message * deleting read-only files * checking existence before setting flags * using unlink * deleting recursively * using pathlib for chmod * using custom workspace * fixed drive * separate handling of single files * simplified read-only handling again * removed vsdevcmd calls as it's already set in the docker Entrypoint * renamed container folders * windows version not needed any more * bumped version number * bumped version number * script cleanup * added cmake parameters for sccache * added sccache * starting sccache with VS environment * added container for Visual Studio 2019 * fixed comments * considering WIndows version * updated to vs2019 * using MS install method * snapshot of VS2019 experiments * using --installRecommended * cleanup of dockerfile * updated path * fixed dockerfile * dumped version number * removed vsdevcmd calls as it's already set in the docker Entrypoint * renamed container folders * windows version not needed any more * bumped version number * bumped version number * script cleanup * removed sccache from vs2017 * making windows image configurable * added versioning * created windows BETA pipeline * added Jenkins label for vs2019 --- .../Jenkinsfile | 169 ++++++++++++++++++ containers/agent-windows-jenkins/Dockerfile | 7 +- .../agent-windows-jenkins/start_agent.ps1 | 2 +- .../Dockerfile | 0 .../VERSION | Bin containers/agent-windows-vs2019/Dockerfile | 66 +++++++ containers/agent-windows-vs2019/VERSION | Bin 0 -> 8 bytes containers/build_deploy.ps1 | 8 +- containers/build_run.ps1 | 14 +- scripts/run_cmake.py | 13 +- scripts/run_ninja.py | 6 - 11 files changed, 258 insertions(+), 27 deletions(-) create mode 100644 Jenkins/BETA-Phabricator-windows-pipeline/Jenkinsfile rename containers/{agent-windows => agent-windows-vs2017}/Dockerfile (100%) rename containers/{agent-windows => agent-windows-vs2017}/VERSION (100%) create mode 100644 containers/agent-windows-vs2019/Dockerfile create mode 100644 containers/agent-windows-vs2019/VERSION diff --git a/Jenkins/BETA-Phabricator-windows-pipeline/Jenkinsfile b/Jenkins/BETA-Phabricator-windows-pipeline/Jenkinsfile new file mode 100644 index 0000000..d60748b --- /dev/null +++ b/Jenkins/BETA-Phabricator-windows-pipeline/Jenkinsfile @@ -0,0 +1,169 @@ +// Copyright 2019 Google LLC +// +// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://llvm.org/LICENSE.txt +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +def success = true + +pipeline { + agent { label 'windows' } + parameters { + string(name: 'DIFF_ID') + string(name: 'PHID') + string(name: 'REV_ID') + } + environment { + CONDUIT_TOKEN = credentials('phabricator-conduit-token') + PHABRICATOR_HOST = 'https://reviews.llvm.org' + PHAB_LOG = "${WORKSPACE}/build/.phabricator-comment" + MY_BUILD_ID = "${JOB_BASE_NAME}-${BUILD_NUMBER}" + RESULT_URL = "https://storage.cloud.google.com/llvm-premerge-checks/results/${MY_BUILD_ID}" + SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts" + // store all build results here, will be uploaded to GCS later + RESULT_DIR = "${WORKSPACE}\\results" + LLVM_DIR = "${WORKSPACE}\\llvm-project" + } + options { + timeout(time:2, unit:'HOURS') + } + stages { + stage("build info"){ + steps { + echo "Building diff ${DIFF_ID} with PHID ${PHID} for Revision ${REV_ID}" + script { + currentBuild.displayName += " D${REV_ID}" + currentBuild.description = "D${REV_ID}" + } + script { + success = true + failure_message = "" + } + } + } + stage("git checkout"){ + steps { + echo "getting llvm-premerge-checks... " + dir("llvm-premerge-checks") + { + git url: 'https://github.com/google/llvm-premerge-checks.git' + } + echo "getting llvm-project... " + dir("llvm-project") + { + git url: 'https://github.com/llvm-premerge-tests/llvm-project.git', branch: "phab-diff-${DIFF_ID}" + powershell 'git clean -fdx' + powershell 'git show -s' + } + // create ${RESULT_DIR} + powershell """ + Remove-Item ${RESULT_DIR} -Recurse -ErrorAction Ignore + New-Item -ItemType Directory -Force -Path ${RESULT_DIR} | Out-Null + """ + } + } + stage('CMake') { + steps { + dir("${LLVM_DIR}"){ + powershell "${SCRIPT_DIR}/run_cmake.py detect" + } + } + post { + failure { + script { + success = false + failure_message = "Failed to run CMake" + } + } + } + } + stage('ninja all') { + steps { + dir("${LLVM_DIR}"){ + powershell "${SCRIPT_DIR}/run_ninja.py all" + } + } + post { + failure { + script { + success = false + failure_message = "Failed to run ninja all" + } + } + } + } + stage('ninja check-all') { + steps { + dir("${LLVM_DIR}"){ + powershell "${SCRIPT_DIR}/run_ninja.py check-all" + } + } + post { + failure { + script { + success = false + failure_message = "Failed to run ninja check-all" + } + } + } + } + } + post { + always { + script { + if (success) { + currentBuild.result = "SUCCESS" + } else { + currentBuild.result = "FAILURE" + } + } + echo "Uploading logs to ${RESULT_URL} ..." + dir("${RESULT_DIR}") { + // gather all result files in a folder + powershell """ + \$ErrorActionPreference = 'Continue' + Write-Host "Getting the console log..." + Invoke-WebRequest -OutFile console-log.txt -uri "http://jenkins.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText" -ErrorAction "Continue" + + Write-Host "Copying build artefacts..." + Copy-Item "${LLVM_DIR}\\choose_projects.log" + Copy-Item "${LLVM_DIR}\\build\\CMakeCache.txt" + Copy-Item "${LLVM_DIR}\\build\\test-results.xml" + """ + } + /// send results to Phabricator + echo "Sending build feedback to Phabricator..." + bat """ + python ${SCRIPT_DIR}/phabtalk/phabtalk.py "${PHID}" "${DIFF_ID}" ^ + --workspace "${LLVM_DIR}" ^ + --conduit-token "${CONDUIT_TOKEN}" ^ + --test-result-file "test-results.xml" ^ + --host "${PHABRICATOR_HOST}/api/" ^ + --results-dir "${RESULT_DIR}" ^ + --results-url "${RESULT_URL}" ^ + --failures "${failure_message}" ^ + --buildresult ${currentBuild.result} ^ + --name "windows" + """ + dir("${RESULT_DIR}") { + // upload results to + // Google Cloud Storage + powershell """ + Write-Host "Uploading results to GCS..." + \$ErrorActionPreference = 'Continue' + gsutil cp *.* gs://llvm-premerge-checks/results/${MY_BUILD_ID}/ + Write-Host "Done." + """ + } + echo "Done." + } + } +} \ No newline at end of file diff --git a/containers/agent-windows-jenkins/Dockerfile b/containers/agent-windows-jenkins/Dockerfile index 454105b..dea7999 100644 --- a/containers/agent-windows-jenkins/Dockerfile +++ b/containers/agent-windows-jenkins/Dockerfile @@ -2,7 +2,7 @@ # use windows server core image ARG agent_windows_version -FROM gcr.io/llvm-premerge-checks/agent-windows:$agent_windows_version +FROM gcr.io/llvm-premerge-checks/agent-windows-vs2019:${agent_windows_version} # install java RUN choco install -y openjdk @@ -18,12 +18,9 @@ RUN powershell -NoProfile -InputFormat None -Command ` RUN pip install gsutil VOLUME C:\credentials -# install python dependencies for the scripts -RUN pip install -r https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/requirements.txt - # temporary directory, can be mounted on host if required VOLUME C:\Temp # start swarm plugin COPY start_agent.ps1 c:\jenkins -CMD powershell c:\jenkins\start_agent.ps1 \ No newline at end of file +ENTRYPOINT ["C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass", "C:\\jenkins\\start_agent.ps1"] diff --git a/containers/agent-windows-jenkins/start_agent.ps1 b/containers/agent-windows-jenkins/start_agent.ps1 index 09b1c59..c27f199 100644 --- a/containers/agent-windows-jenkins/start_agent.ps1 +++ b/containers/agent-windows-jenkins/start_agent.ps1 @@ -29,5 +29,5 @@ java -jar ${env:SWARM_PLUGIN_JAR} ` -master http://${JENKINS_SERVER}:8080 ` -executors 1 ` -fsroot ${AGENT_ROOT} ` - -labels windows ` + -labels "windows vs2019" ` -name ${env:PARENT_HOSTNAME} \ No newline at end of file diff --git a/containers/agent-windows/Dockerfile b/containers/agent-windows-vs2017/Dockerfile similarity index 100% rename from containers/agent-windows/Dockerfile rename to containers/agent-windows-vs2017/Dockerfile diff --git a/containers/agent-windows/VERSION b/containers/agent-windows-vs2017/VERSION similarity index 100% rename from containers/agent-windows/VERSION rename to containers/agent-windows-vs2017/VERSION diff --git a/containers/agent-windows-vs2019/Dockerfile b/containers/agent-windows-vs2019/Dockerfile new file mode 100644 index 0000000..3820a5e --- /dev/null +++ b/containers/agent-windows-vs2019/Dockerfile @@ -0,0 +1,66 @@ +# escape=` +FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2019 + +# Restore the default Windows shell for correct batch processing. +SHELL ["cmd", "/S", "/C"] + +# Download the Build Tools bootstrapper. +ADD https://aka.ms/vs/16/release/vs_buildtools.exe C:\TEMP\vs_buildtools.exe + +# Download channel for fixed install. +ARG CHANNEL_URL=https://aka.ms/vs/16/release/channel +ADD ${CHANNEL_URL} C:\TEMP\VisualStudio.chman + +# Install Build Tools with C++ workload. +# - Documentation for docker installation +# https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2019 +# - Documentation on workloads +# https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools?view=vs-2019#c-build-tools +# - Documentation on flags +# https://docs.microsoft.com/en-us/visualstudio/install/use-command-line-parameters-to-install-visual-studio?view=vs-2019 +RUN C:\TEMP\vs_buildtools.exe --quiet --wait --norestart --nocache ` + --channelUri C:\TEMP\VisualStudio.chman ` + --installChannelUri C:\TEMP\VisualStudio.chman ` + --installPath C:\BuildTools ` + --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended ` + || IF "%ERRORLEVEL%"=="3010" EXIT 0 + +# install chocolately as package manager +RUN powershell -NoProfile -InputFormat None -Command ` + iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')) ; ` + choco feature disable --name showDownloadProgress + +# install tools as described in https://llvm.org/docs/GettingStartedVS.html +# and a few more that were not documented... +RUN choco install -y ninja +RUN choco install -y git +RUN choco install -y cmake --version 3.15.4 +RUN choco install -y python3 +RUN choco install -y gnuwin +RUN choco install -y sccache +# set local cache folder for sccache +ENV SCCACHE_DIR=C:\ws\sccache +RUN pip install psutil + +# install python dependencies for the scripts +RUN pip install -r https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/requirements.txt + +# configure Python encoding +ENV PYTHONIOENCODING=UTF-8 + +# update the path variable +RUN powershell -NoProfile -InputFormat None -Command ` + $path = $env:path + ';c:\Program Files (x86)\GnuWin32\bin;C:\Program Files\CMake\bin'; ` + Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment\' -Name Path -Value $path + +# use this folder to store the worksapce' +VOLUME C:\ws +WORKDIR C:\ws + +# support long file names during git checkout +RUN git config --system core.longpaths true & ` + git config --global core.autocrlf false + +# Define the entry point for the docker container. +# This entry point starts the developer command prompt and launches the PowerShell shell. +ENTRYPOINT ["C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"] diff --git a/containers/agent-windows-vs2019/VERSION b/containers/agent-windows-vs2019/VERSION new file mode 100644 index 0000000000000000000000000000000000000000..c7250cbdc15f21d024aa64ed135ba73e05b97b58 GIT binary patch literal 8 PcmezW&xnDSfr|kE5M}~L literal 0 HcmV?d00001 diff --git a/containers/build_deploy.ps1 b/containers/build_deploy.ps1 index 8c0608d..7df615a 100644 --- a/containers/build_deploy.ps1 +++ b/containers/build_deploy.ps1 @@ -34,21 +34,15 @@ $QUALIFIED_NAME="${GCR_HOSTNAME}/${GCP_PROJECT}/${IMAGE_NAME}" Push-Location "$PSScriptRoot\$IMAGE_NAME" $container_version=[int](Get-Content $VERSION_FILE) $container_version+=1 -$agent_windows_version=Get-Content "../agent-windows/$VERSION_FILE" +$agent_windows_version=Get-Content "../agent-windows-vs2019/$VERSION_FILE" Write-Host "Building ${IMAGE_NAME}:${container_version}..." Write-Host "Using windows-agent ${agent_windows_version}" -# TODO: get current Windows version number from host via "cmd /c ver" -# to solve these issues: https://stackoverflow.com/questions/43123851/unable-to-run-cygwin-in-windows-docker-container/52273489#52273489 -$windows_version="10.0.17763.1039" -Write-Host "Using windows version ${windows_version}" - Invoke-Call -ScriptBlock { docker build . ` -t ${IMAGE_NAME}:${container_version} ` -t ${IMAGE_NAME}:latest ` - --build-arg windows_version=$windows_version ` --build-arg agent_windows_version=$agent_windows_version } Invoke-Call -ScriptBlock { diff --git a/containers/build_run.ps1 b/containers/build_run.ps1 index 1c3e5ca..136aec6 100644 --- a/containers/build_run.ps1 +++ b/containers/build_run.ps1 @@ -21,15 +21,25 @@ param( # set script to stop on first error $ErrorActionPreference = "Stop" +$agent_windows_version=Get-Content "../agent-windows-vs2019/VERSION" # some docs recommend setting 2GB memory limit - docker build --memory 2GB -t $IMAGE_NAME --build-arg token=$token "$PSScriptRoot\$IMAGE_NAME" +docker build ` + --memory 2GB ` + -t $IMAGE_NAME ` + --build-arg token=$token ` + --build-arg agent_windows_version=$agent_windows_version ` + "$PSScriptRoot\$IMAGE_NAME" If ($LastExitCode -ne 0) { exit } # mount a persistent workspace for experiments -docker run -it -v D:\:C:\ws -v C:\credentials:C:\credentials -e PARENT_HOSTNAME=$env:computername $IMAGE_NAME $CMD +docker run -it ` + -v D:\:C:\ws ` + -v C:\credentials:C:\credentials ` + -e PARENT_HOSTNAME=$env:computername ` + $IMAGE_NAME $CMD If ($LastExitCode -ne 0) { exit } \ No newline at end of file diff --git a/scripts/run_cmake.py b/scripts/run_cmake.py index 06a2f07..e22256f 100755 --- a/scripts/run_cmake.py +++ b/scripts/run_cmake.py @@ -101,8 +101,14 @@ def _create_args(config: Configuration, llvm_enable_projects: str) -> List[str]: arguments.extend(config.general_cmake_arguments) arguments.extend(config.specific_cmake_arguments) + # enable sccache + if 'SCCACHE_DIR' in os.environ: + arguments.extend([ + '-DCMAKE_C_COMPILER_LAUNCHER=sccache', + '-DCMAKE_CXX_COMPILER_LAUNCHER=sccache', + ]) # enable ccache if the path is set in the environment - if 'CCACHE_PATH' in os.environ: + elif 'CCACHE_PATH' in os.environ: arguments.extend([ '-D LLVM_CCACHE_BUILD=ON', '-D LLVM_CCACHE_DIR={}'.format(os.environ['CCACHE_PATH']), @@ -132,11 +138,6 @@ def run_cmake(projects: str, repo_path: str, config_file_path: str = None, *, dr arguments = _create_args(config, llvm_enable_projects) cmd = 'cmake ' + ' '.join(arguments) - # On Windows: configure Visutal Studio before running cmake - if config.operating_system == OperatingSystem.Windows: - # FIXME: move this path to a config file - cmd = r'"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64 && ' + cmd - print('Running cmake with these arguments:\n{}'.format(cmd)) if dryrun: print('Dryrun, not invoking CMake!') diff --git a/scripts/run_ninja.py b/scripts/run_ninja.py index 16b125f..16972b5 100755 --- a/scripts/run_ninja.py +++ b/scripts/run_ninja.py @@ -21,12 +21,6 @@ import subprocess def run_ninja(target: str, repo_path: str): build_dir = os.path.join(repo_path, 'build') cmd = 'ninja {}'.format(target) - - # On Windows: configure Visutal Studio before running ninja - if platform.system() == 'Windows': - # FIXME: move this path to a config file - cmd = r'"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64 && ' + cmd - subprocess.check_call(cmd, shell=True, cwd=build_dir)