revi-archive/llvm-premerge-checks
1
0
Fork 0
Code Activity

update agent-windows-buildkite

Browse source 
add comment on images
This commit is contained in:
Mikhail Goncharov 2023-03-28 13:35:04 +02:00
parent 3145164ee7
commit 609ed91cc6
4 changed files with 21 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
containers/agent-windows-buildkite/Dockerfile
View file

@ -1,6 +1,8 @@
# escape=` # escape=`
FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2019 FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2019
# This docker image is run on manually maintained windows machines in 'llvm-premerge-check' cluster.
# Restore the default Windows shell for correct batch processing. # Restore the default Windows shell for correct batch processing.
SHELL ["cmd", "/S", "/C"] SHELL ["cmd", "/S", "/C"]
@ -12,9 +14,9 @@ ARG CHANNEL_URL=https://aka.ms/vs/16/release/channel
ADD ${CHANNEL_URL} C:\TEMP\VisualStudio.chman ADD ${CHANNEL_URL} C:\TEMP\VisualStudio.chman
# Install Build Tools with C++ workload. # Install Build Tools with C++ workload.
# - Documentation for docker installation # - Documentation for docker installation
# https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2019 # https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2019
# - Documentation on workloads # - Documentation on workloads
# https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools?view=vs-2019#c-build-tools # https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools?view=vs-2019#c-build-tools
# - Documentation on flags # - Documentation on flags
# https://docs.microsoft.com/en-us/visualstudio/install/use-command-line-parameters-to-install-visual-studio?view=vs-2019 # https://docs.microsoft.com/en-us/visualstudio/install/use-command-line-parameters-to-install-visual-studio?view=vs-2019
@ -27,7 +29,7 @@ RUN C:\TEMP\vs_buildtools.exe --quiet --wait --norestart --nocache `
--includeRecommended ` --includeRecommended `
|| IF "%ERRORLEVEL%"=="3010" EXIT 0 || IF "%ERRORLEVEL%"=="3010" EXIT 0
# Register DIA dll (Debug Interface Access) so it can be used to symbolize # Register DIA dll (Debug Interface Access) so it can be used to symbolize
# the stack traces. Register dll for 32 and 64 bit. # the stack traces. Register dll for 32 and 64 bit.
# see https://developercommunity.visualstudio.com/content/problem/290674/msdia140dll-is-not-registered-on-vs2017-hosts.html # see https://developercommunity.visualstudio.com/content/problem/290674/msdia140dll-is-not-registered-on-vs2017-hosts.html
RUN regsvr32 /S "C:\BuildTools\DIA SDK\bin\amd64\msdia140.dll" & ` RUN regsvr32 /S "C:\BuildTools\DIA SDK\bin\amd64\msdia140.dll" & `
@ -65,15 +67,15 @@ RUN pip install psutil
# install python dependencies for the scripts # install python dependencies for the scripts
RUN pip install -r https://raw.githubusercontent.com/google/llvm-premerge-checks/main/scripts/requirements.txt RUN pip install -r https://raw.githubusercontent.com/google/llvm-premerge-checks/main/scripts/requirements.txt
RUN curl -LO https://github.com/mstorsjo/llvm-mingw/releases/download/20220323/llvm-mingw-20220323-ucrt-x86_64.zip && ` RUN curl -LO https://github.com/mstorsjo/llvm-mingw/releases/download/20230320/llvm-mingw-20230320-ucrt-x86_64.zip && `
powershell Expand-Archive llvm-mingw-*-ucrt-x86_64.zip -DestinationPath . && ` powershell Expand-Archive llvm-mingw-*-ucrt-x86_64.zip -DestinationPath . && `
del llvm-mingw-*-ucrt-x86_64.zip && ` del llvm-mingw-*-ucrt-x86_64.zip && `
ren llvm-mingw-20220323-ucrt-x86_64 llvm-mingw ren llvm-mingw-20230320-ucrt-x86_64 llvm-mingw
# configure Python encoding # configure Python encoding
ENV PYTHONIOENCODING=UTF-8 ENV PYTHONIOENCODING=UTF-8
# update the path variable # update the path variable
# C:\Program Files\Git\usr\bin contains a usable bash and other unix tools. # C:\Program Files\Git\usr\bin contains a usable bash and other unix tools.
# C:\llvm-mingw\bin contains Clang configured for mingw targets and # C:\llvm-mingw\bin contains Clang configured for mingw targets and
# corresponding sysroots. Both the 'llvm' package (with Clang defaulting # corresponding sysroots. Both the 'llvm' package (with Clang defaulting

102
containers/agent-windows-vs2019/Dockerfile
View file

@ -1,102 +0,0 @@
# escape=`
FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2019
# Restore the default Windows shell for correct batch processing.
SHELL ["cmd", "/S", "/C"]
# Download the Build Tools bootstrapper.
ADD https://aka.ms/vs/16/release/vs_buildtools.exe C:\TEMP\vs_buildtools.exe
# Download channel for fixed install.
ARG CHANNEL_URL=https://aka.ms/vs/16/release/channel
ADD ${CHANNEL_URL} C:\TEMP\VisualStudio.chman
# Install Build Tools with C++ workload.
# - Documentation for docker installation
# https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2019
# - Documentation on workloads
# https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools?view=vs-2019#c-build-tools
# - Documentation on flags
# https://docs.microsoft.com/en-us/visualstudio/install/use-command-line-parameters-to-install-visual-studio?view=vs-2019
RUN C:\TEMP\vs_buildtools.exe --quiet --wait --norestart --nocache `
--channelUri C:\TEMP\VisualStudio.chman `
--installChannelUri C:\TEMP\VisualStudio.chman `
--installPath C:\BuildTools `
--add Microsoft.VisualStudio.Workload.VCTools `
--add Microsoft.VisualStudio.Component.VC.ATL `
--includeRecommended `
|| IF "%ERRORLEVEL%"=="3010" EXIT 0
# Register DIA dll (Debug Interface Access) so it can be used to symbolize
# the stack traces. Register dll for 32 and 64 bit.
# see https://developercommunity.visualstudio.com/content/problem/290674/msdia140dll-is-not-registered-on-vs2017-hosts.html
RUN regsvr32 /S "C:\BuildTools\DIA SDK\bin\amd64\msdia140.dll" & `
regsvr32 /S "C:\BuildTools\DIA SDK\bin\msdia140.dll"
# install chocolately as package manager
RUN powershell -NoProfile -InputFormat None -Command `
iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')) ; `
choco feature disable --name showDownloadProgress
# install tools as described in https://llvm.org/docs/GettingStartedVS.html
# and a few more that were not documented...
RUN choco install -y ninja git
# Pin an older version of Python; the current Python 3.10 fails when
# doing "pip install" for the other dependencies, as it fails to find libxml
# while compiling some package.
RUN choco install -y python3 --version 3.9.7
# ActivePerl is currently not installable via Chocolatey, see
# http://disq.us/p/2ipditb. Install StrawberryPerl instead. Unfortunately,
# StrawberryPerl not only installs Perl, but also a redundant C/C++ compiler
# toolchain, and a copy of pkg-config which can cause misdetections for other
# built products, see
# https://github.com/StrawberryPerl/Perl-Dist-Strawberry/issues/11 for further
# details. Remove the redundant and unnecessary parts of the StrawberryPerl
# install.
RUN choco install -y strawberryperl && `
rmdir /q /s c:\strawberry\c && `
del /q c:\strawberry\perl\bin\pkg-config*
# libcxx requires clang(-cl) to be available
RUN choco install -y sccache llvm
RUN pip install psutil
# install python dependencies for the scripts
RUN pip install -r https://raw.githubusercontent.com/google/llvm-premerge-checks/main/scripts/requirements.txt
RUN curl -LO https://github.com/mstorsjo/llvm-mingw/releases/download/20230320/llvm-mingw-20230320-ucrt-x86_64.zip && `
powershell Expand-Archive llvm-mingw-*-ucrt-x86_64.zip -DestinationPath . && `
del llvm-mingw-*-ucrt-x86_64.zip && `
ren llvm-mingw-20230320-ucrt-x86_64 llvm-mingw
# configure Python encoding
ENV PYTHONIOENCODING=UTF-8
# update the path variable
# C:\Program Files\Git\usr\bin contains a usable bash and other unix tools.
# C:\llvm-mingw\bin contains Clang configured for mingw targets and
# corresponding sysroots. Both the 'llvm' package (with Clang defaulting
# to MSVC targets) and this directory contains executables named
# 'clang.exe' - add this last to let the other one have precedence.
# To use these compilers, use the triple prefixed form, e.g.
# x86_64-w64-mingw32-clang.
RUN powershell -Command `
[System.Environment]::SetEnvironmentVariable('PATH', `
[System.Environment]::GetEnvironmentVariable('PATH', 'machine') + ';C:\Program Files\Git\usr\bin;C:\llvm-mingw\bin', `
'machine')
# use this folder to store the worksapce'
VOLUME C:\ws
WORKDIR C:\ws
# support long file names during git checkout
RUN git config --system core.longpaths true & `
git config --global core.autocrlf false
# Define the entry point for the docker container.
# This entry point starts the developer command prompt and launches the PowerShell shell.
#
# For running manually:
# C:\BuildTools\Common7\Tools\VsDevCmd.bat -arch=amd64 -host_arch=amd64
ENTRYPOINT ["C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "-arch=amd64", "-host_arch=amd64", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"]

2
containers/buildkite-premerge-windows/Dockerfile
View file

@ -1,5 +1,7 @@
# escape=` # escape=`
# This experimental Windows container is meant to be used in Terraform deployment.
# Agent image for LLVM org cluster. # Agent image for LLVM org cluster.
# maybe mcr.microsoft.com/windows/servercore:ltsc2022? # maybe mcr.microsoft.com/windows/servercore:ltsc2022?
FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2019 FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2019

23
docs/development.md
View file

@ -35,7 +35,7 @@ Phabricator side these things were configured
- Herald [rule for everyone](https://reviews.llvm.org/H576) and for [beta - Herald [rule for everyone](https://reviews.llvm.org/H576) and for [beta
testers](https://reviews.llvm.org/H511). Note that right now there is no testers](https://reviews.llvm.org/H511). Note that right now there is no
difference between beta and "normal" builds. difference between beta and "normal" builds.
- the [merge_guards_bot user](https://reviews.llvm.org/p/merge_guards_bot/) - the [merge_guards_bot user](https://reviews.llvm.org/p/merge_guards_bot/)
account for writing comments. account for writing comments.
@ -44,7 +44,7 @@ account for writing comments.
Buildkite allows [dynamically define pipelines as the output of a Buildkite allows [dynamically define pipelines as the output of a
command](https://buildkite.com/docs/pipelines/defining-steps#dynamic-pipelines). command](https://buildkite.com/docs/pipelines/defining-steps#dynamic-pipelines).
That gives us the flexibility to generate pipeline code using the code from a That gives us the flexibility to generate pipeline code using the code from a
specific branch of pre-merge checks. Thus, specific branch of pre-merge checks. Thus,
[changes can be tested](./playbooks.md#testing-changes-before-merging) [changes can be tested](./playbooks.md#testing-changes-before-merging)
before affecting everyone. before affecting everyone.
@ -101,21 +101,21 @@ HTTP authentication and forwards all requests from load balancer to
Follow up to date docs to install [reverse Follow up to date docs to install [reverse
proxy](https://kubernetes.github.io/ingress-nginx/deploy/#gce-gke). proxy](https://kubernetes.github.io/ingress-nginx/deploy/#gce-gke).
[cert-manager] is installed with helm https://cert-manager.io/docs/installation/helm/ [cert-manager] is installed with helm https://cert-manager.io/docs/installation/helm/
helm install \ helm install \
cert-manager jetstack/cert-manager \ cert-manager jetstack/cert-manager \
--namespace cert-manager \ --namespace cert-manager \
--create-namespace \ --create-namespace \
--version v1.9.1 \ --version v1.9.1 \
--set installCRDs=true --set installCRDs=true
We also have [certificate manager](https://cert-manager.io/docs/) and We also have [certificate manager](https://cert-manager.io/docs/) and
[lets-encrypt configuration](../kubernetes/cert-issuer.yaml) in place, but they are [lets-encrypt configuration](../kubernetes/cert-issuer.yaml) in place, but they are
not used at the moment and should be removed if we decide to live with static IP. not used at the moment and should be removed if we decide to live with static IP.
HTTP auth is configured with k8s secret 'http-auth' in 'buildkite' namespace HTTP auth is configured with k8s secret 'http-auth' in 'buildkite' namespace
(see [how to update auth](playbooks.md#update-http-auth-credentials)). (see [how to update auth](playbooks.md#update-http-auth-credentials)).
## Linux agents ## Linux agents
@ -156,11 +156,10 @@ debugged, updated, and scaled easily:
- [Linux](../containers/buildkite-premerge-debian/Dockerfile). We use - [Linux](../containers/buildkite-premerge-debian/Dockerfile). We use
[Kubernetes deployment](../kubernetes/buildkite) to manage these agents. [Kubernetes deployment](../kubernetes/buildkite) to manage these agents.
- [Windows](../containers/agent-windows-buildkite/Dockerfile) based on [Windows - [Windows](../containers/agent-windows-buildkite/Dockerfile). At the moment they are run as
vs2019](../containers/agent-windows-vs2019). At the moment they are run as
multiple individual VM instances. multiple individual VM instances.
See [playbooks](playbooks.md) how to manage and set up machines. See [playbooks](playbooks.md) how to manage and set up machines.
# Compilation caching # Compilation caching