From de34fb9572ec220130d79a429f9e677177389de2 Mon Sep 17 00:00:00 2001
From: ChristianKuehnel <kuhnel@google.com>
Date: Tue, 26 Nov 2019 17:10:30 +0100
Subject: [PATCH] Create docker container for Windows build (#52)

* first steps towards windows builds #25

* added build scripts for windows

* moved from powershell to CMD

* script cleanup

* set up jenkins swarm plugin

* added deployment script for windows

* first draft of jenkins file for windows

* first shot at windows pods

* first draft for buildkite agent

* added call to run_cmake.bat

* added call to run_ninja.bat

* added new buildkite script

* added Dockerfile for buildkite agent

* fixed userprofile path

* added /IncludeOptional for VS

I hope that includes all required libraries

* moved work folder to W:

* added git config for long file names

* added more CMake flags

* setting x64 bit arch in a different way

* and yet another way to set 64bit

* merged buildkite changes to jenkins scripts

* hiding license in batch script runs

* running release build

* added test xml log

* compilation is working, but not all tests

* cmake is in patch now

* updated the dependencies from buildkite config

* dockerfile cleanup

* change how git handles line endings

* updated comment

* performance improvment for volume

* changed workdir folder

* using gnuwin package

split installation into separate layers to speed up incremental changes
---
 Jenkins/Windows-pipeline/Jenkinsfile          | 68 +++++++++++++++++++
 containers/README.md                          |  9 +++
 containers/agent-windows-buildkite/Dockerfile | 56 +++++++++++++++
 containers/agent-windows/Dockerfile           | 44 ++++++++++++
 containers/build_deploy.ps1                   | 33 +++++++++
 containers/build_run.ps1                      | 35 ++++++++++
 kubernetes/cluster_create.sh                  |  5 ++
 kubernetes/jenkins.yaml                       | 39 +++++++++++
 scripts/benchmark.ps1                         | 24 +++++++
 scripts/run_buildkite.bat                     | 25 +++++++
 scripts/run_cmake.bat                         | 27 ++++++++
 scripts/run_ninja.bat                         | 22 ++++++
 12 files changed, 387 insertions(+)
 create mode 100644 Jenkins/Windows-pipeline/Jenkinsfile
 create mode 100644 containers/agent-windows-buildkite/Dockerfile
 create mode 100644 containers/agent-windows/Dockerfile
 create mode 100644 containers/build_deploy.ps1
 create mode 100644 containers/build_run.ps1
 create mode 100644 scripts/benchmark.ps1
 create mode 100644 scripts/run_buildkite.bat
 create mode 100644 scripts/run_cmake.bat
 create mode 100644 scripts/run_ninja.bat

diff --git a/Jenkins/Windows-pipeline/Jenkinsfile b/Jenkins/Windows-pipeline/Jenkinsfile
new file mode 100644
index 0000000..e41b9c0
--- /dev/null
+++ b/Jenkins/Windows-pipeline/Jenkinsfile
@@ -0,0 +1,68 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://llvm.org/LICENSE.txt
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pipeline {
+    agent any
+    triggers {
+        pollSCM 'H H/2 * * *'
+    }
+    environment {
+        BUILD_ID="${JOB_BASE_NAME}-${BUILD_NUMBER}"
+        //TARGET_DIR="/mnt/nfs/results/${BUILD_ID}"
+        SCRIPT_DIR = "${WORKSPACE}\\llvm-premerge-checks\\scripts"
+
+    }
+    stages {
+        stage("git checkout"){
+            steps {
+                git url: 'https://github.com/llvm/llvm-project.git'
+                sh 'git clean -fdx'
+                sh 'mkdir -p llvm-premerge-checks'
+                dir("llvm-premerge-checks")
+                {
+                    git url: 'https://github.com/google/llvm-premerge-checks.git'
+                }
+            }            
+        }
+        stage('CMake') {
+            steps {
+                sh "${SCRIPT_DIR}\\run_cmake.bat"
+            }
+        }
+        stage('ninja all') {
+            steps {
+                sh "${SCRIPT_DIR}\\run_ninja.bat all"
+            }
+        }
+        stage('ninja check') {
+            steps {
+                sh "${SCRIPT_DIR}\\run_ninja.bat check"
+            }
+        }
+        stage('ninja check-all') {
+            steps {
+                sh "${SCRIPT_DIR}\\run_ninja.bat check-all"
+            }
+        }
+    }
+    // post { 
+    //     always { 
+    //         echo "Console log is available at https://results.new.llvm-merge-guard.org/${BUILD_ID}"
+    //         dir("${env.TARGET_DIR}") {
+    //             // copy console log to result folder
+    //             sh "wget -qO console-log.txt http://jenkins-ui.jenkins.svc.cluster.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText"
+    //         }
+    //     }
+    // }
+}
\ No newline at end of file
diff --git a/containers/README.md b/containers/README.md
index 1f0fd27..d054770 100644
--- a/containers/README.md
+++ b/containers/README.md
@@ -1,2 +1,11 @@
 # Overview
 This folder contains files related to the machines running the build/test/checks for the merge guards.
+
+## Scripts
+The scripts are written in bash (for Linux) and powershell (for Windows).
+
+### build_run.(sh|ps1)
+Build the docker image and run it locally. This is useful for testing it.
+
+### build_deploy.(sh|ps1)
+Build the docker and deploy it to the GCP registry. This is useful for deploying it in the Kubernetes cluster.
\ No newline at end of file
diff --git a/containers/agent-windows-buildkite/Dockerfile b/containers/agent-windows-buildkite/Dockerfile
new file mode 100644
index 0000000..1e643c1
--- /dev/null
+++ b/containers/agent-windows-buildkite/Dockerfile
@@ -0,0 +1,56 @@
+# escape=`
+
+# use windows server core image
+ARG version=ltsc2019
+FROM mcr.microsoft.com/windows/servercore:$version
+
+# install chocolately as package manager
+RUN powershell -Command `
+    iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')) ; `
+    choco feature disable --name showDownloadProgress
+
+# install Visual Studio build tools
+RUN powershell -NoProfile -InputFormat None -Command `
+    choco install visualcpp-build-tools `
+        --version 15.0.26228.20170424 -y --params "'/IncludeOptional'" ;`
+    Write-Host 'Waiting for Visual C++ Build Tools to finish'; `
+    Wait-Process -Name vs_installer
+
+# install other tools as described in https://llvm.org/docs/GettingStartedVS.html
+# and a few more that were not documented...
+RUN choco install -y git & `
+    choco install -y cmake --version 3.15.4 & `
+    choco install -y python2 & `
+    choco install -y gnuwin32-coreutils.install & `
+    choco install -y ninja & `
+    choco install -y grep & ` 
+    choco install -y sed & ` 
+    choco install -y diffutils
+RUN pip install psutil    
+
+# configure Python encoding
+ENV PYTHONIOENCODING=UTF-8
+
+# update the path variable    
+RUN powershell.exe -Command $path = $env:path + ';c:\Program Files (x86)\GnuWin32\bin;C:\Program Files\CMake\bin'; Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment\' -Name Path -Value $path
+
+# use this drive to store the worksapce'
+VOLUME W:
+
+# install buildkite agent as described in the documentation
+# https://buildkite.com/docs/agent/v3/windows
+# Supply your agent token via the arguement "-token <mytoken" when building the image
+ARG token
+ENV buildkiteAgentToken="${token}"
+ENV BUILDKITE_BUILD_PATH=W:\buildkite
+ENV BUILDKITE_AGENT_NAME=amd64-windows-visualstudio2017
+
+RUN powershell -NoProfile -InputFormat None -Command `
+    Set-ExecutionPolicy Bypass -Scope Process -Force ; `
+    iex ((New-Object System.Net.WebClient).DownloadString('https://raw.githubusercontent.com/buildkite/agent/master/install.ps1'))
+
+# support long file names and do not change line endings
+RUN git config --system core.longpaths true & `
+    git config --global core.autocrlf false
+
+CMD "C:\buildkite-agent\bin\buildkite-agent.exe" start
\ No newline at end of file
diff --git a/containers/agent-windows/Dockerfile b/containers/agent-windows/Dockerfile
new file mode 100644
index 0000000..c9d77a3
--- /dev/null
+++ b/containers/agent-windows/Dockerfile
@@ -0,0 +1,44 @@
+# escape=`
+
+# use windows server core image
+ARG version=ltsc2019
+FROM mcr.microsoft.com/windows/servercore:$version
+
+# install chocolately as package manager
+RUN powershell -Command `
+    iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')) ; `
+    choco feature disable --name showDownloadProgress
+
+# install Visual Studio build tools
+RUN powershell -NoProfile -InputFormat None -Command `
+    choco install visualcpp-build-tools `
+        --version 15.0.26228.20170424 -y --params "'/IncludeOptional'" ;`
+    Write-Host 'Waiting for Visual C++ Build Tools to finish'; `
+    Wait-Process -Name vs_installer
+
+# install other tools as described in https://llvm.org/docs/GettingStartedVS.html
+# and a few more that were not documented...
+RUN choco install -y git
+RUN choco install -y cmake --version 3.15.4
+RUN choco install -y python2
+RUN choco install -y ninja
+RUN choco install -y gnuwin
+RUN pip install psutil    
+
+# configure Python encoding
+ENV PYTHONIOENCODING=UTF-8
+
+# update the path variable    
+RUN powershell.exe -Command $path = $env:path + ';c:\Program Files (x86)\GnuWin32\bin;C:\Program Files\CMake\bin'; Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment\' -Name Path -Value $path
+
+# use this folder to store the worksapce'
+VOLUME C:\workspace
+WORKDIR C:\workspace
+
+# support long file names during git checkout
+RUN git config --system core.longpaths true & `
+    git config --global core.autocrlf false
+
+# Start developer command prompt with any other commands specified.
+#ENTRYPOINT "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat" &
+CMD cmd
\ No newline at end of file
diff --git a/containers/build_deploy.ps1 b/containers/build_deploy.ps1
new file mode 100644
index 0000000..7b6f374
--- /dev/null
+++ b/containers/build_deploy.ps1
@@ -0,0 +1,33 @@
+# Copyright 2019 Google LLC
+
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://llvm.org/LICENSE.txt
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# define command line arguments
+param(
+    [Parameter(Mandatory=$true)][string]$IMAGE_NAME
+)
+
+New-Variable -Name ROOT_DIR -Value (Get-Item $PSScriptRoot).Parent.FullName
+
+# get config options
+Get-Content "${ROOT_DIR}/k8s_config" | Foreach-Object{
+    $var = $_.Split('=')
+    New-Variable -Name $var[0] -Value $var[1]
+}
+
+New-Variable -Name QUALIFIED_NAME -Value "${GCR_HOSTNAME}/${GCP_PROJECT}/${IMAGE_NAME}"
+
+Set-Location "$PSScriptRoot\$IMAGE_NAME"
+docker build -t $IMAGE_NAME .
+docker tag $IMAGE_NAME $QUALIFIED_NAME
+docker push $QUALIFIED_NAME
\ No newline at end of file
diff --git a/containers/build_run.ps1 b/containers/build_run.ps1
new file mode 100644
index 0000000..b9464ed
--- /dev/null
+++ b/containers/build_run.ps1
@@ -0,0 +1,35 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# define command line arguments
+param(
+    [Parameter(Mandatory=$true)][string]$IMAGE_NAME,
+    [string]$token
+)
+
+# set script to stop on first error
+$ErrorActionPreference = "Stop"
+
+# some docs recommend setting 2GB memory limit
+ docker build --memory 2GB -t $IMAGE_NAME --build-arg token=$token "$PSScriptRoot\$IMAGE_NAME"
+If ($LastExitCode -ne 0) {
+    exit
+}
+
+# mount a persistent workspace for experiments
+New-Item -ItemType Directory -Force -Path $Env:USERPROFILE\workspace
+docker run -it -v $Env:USERPROFILE\workspace:C:\workspace $IMAGE_NAME
+If ($LastExitCode -ne 0) {
+    exit
+}
\ No newline at end of file
diff --git a/kubernetes/cluster_create.sh b/kubernetes/cluster_create.sh
index f8bdf02..397af2c 100755
--- a/kubernetes/cluster_create.sh
+++ b/kubernetes/cluster_create.sh
@@ -31,6 +31,11 @@ gcloud container clusters create $GCP_CLUSTER --zone $GCP_ZONE \
 gcloud container node-pools create jenkins-agents --cluster $GCP_CLUSTER --zone $GCP_ZONE \
     --machine-type=n1-standard-32 --num-nodes=2 --local-ssd-count=1
 
+# Windows agents with local ssd
+gcloud container node-pools create windows-pool --cluster $GCP_CLUSTER \
+    --image-type=WINDOWS_SAC --no-enable-autoupgrade \
+    --machine-type=n1-standard-16 --local-ssd-count=1
+
 # create static IP address
 # IP can be created, but not used in Ingress. Not sure why
 gcloud compute addresses create web-static-ip --zone=$GCP_ZONE
\ No newline at end of file
diff --git a/kubernetes/jenkins.yaml b/kubernetes/jenkins.yaml
index a990e67..65f62cd 100644
--- a/kubernetes/jenkins.yaml
+++ b/kubernetes/jenkins.yaml
@@ -186,5 +186,44 @@ spec:
           # directory location on host
           path: /mnt/disks/ssd0
           type: Directory          
+      nodeSelector:
+        cloud.google.com/gke-nodepool: jenkins-agents
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: agent-windows-visualstudio2017-ssd
+  namespace: jenkins
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: agent-windows-visualstudio2017-ssd
+    spec:
+      containers:
+      - name: agent-windows-visualstudio2017-ssd
+        image: gcr.io/llvm-premerge-checks/agent-windows
+        resources:
+          limits:
+            cpu: 14
+            memory: 20Gi
+          requests:
+            cpu: 14
+            memory: 20Gi
+#        volumeMounts:
+#        - name: nfs-pvc
+#          mountPath: /mnt/nfs          
+#        - name: ssd 
+#          mountPath: /mnt/disks/ssd0
+#      volumes:
+#      - name: nfs-pvc
+#        persistentVolumeClaim:
+#          claimName: nfs-jenkins
+#      - name: ssd
+#        hostPath:
+#          # directory location on host
+#          path: /mnt/disks/ssd0
+#          type: Directory          
       nodeSelector:
         cloud.google.com/gke-nodepool: jenkins-agents
\ No newline at end of file
diff --git a/scripts/benchmark.ps1 b/scripts/benchmark.ps1
new file mode 100644
index 0000000..91b52f3
--- /dev/null
+++ b/scripts/benchmark.ps1
@@ -0,0 +1,24 @@
+# Copyright 2019 Google LLC
+
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://llvm.org/LICENSE.txt
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+param(
+    [Parameter(Mandatory=$true)][string]$WORKDIR
+)
+
+Measure-Command { git clone --depth 1 https://github.com/llvm/llvm-project $WORKDIR }
+
+Set-Location $WORKDIR
+Measure-Command {& $PSScriptRoot\run_cmake.bat}
+Measure-Command {& $PSScriptRoot\run_ninja.bat all}
+Measure-Command {& $PSScriptRoot\run_ninja.bat check-all}
diff --git a/scripts/run_buildkite.bat b/scripts/run_buildkite.bat
new file mode 100644
index 0000000..56423bc
--- /dev/null
+++ b/scripts/run_buildkite.bat
@@ -0,0 +1,25 @@
+rem Copyright 2019 Google LLC
+rem
+rem Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+rem you may not use this file except in compliance with the License.
+rem You may obtain a copy of the License at
+rem
+rem     https://llvm.org/LICENSE.txt
+rem
+rem Unless required by applicable law or agreed to in writing, software
+rem distributed under the License is distributed on an "AS IS" BASIS,
+rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+rem See the License for the specific language governing permissions and
+rem limitations under the License.
+md build
+cd build
+
+call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64
+
+cmake.exe ..\llvm -G Ninja ^
+    -D LLVM_ENABLE_PROJECTS="clang;clang-tools-extra;libcxx;libcxxabi;lld" ^
+    -D LLVM_ENABLE_ASSERTIONS=ON
+
+ninja all
+
+ninja check-all
\ No newline at end of file
diff --git a/scripts/run_cmake.bat b/scripts/run_cmake.bat
new file mode 100644
index 0000000..17056a8
--- /dev/null
+++ b/scripts/run_cmake.bat
@@ -0,0 +1,27 @@
+@rem Copyright 2019 Google LLC
+@rem
+@rem Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem     https://llvm.org/LICENSE.txt
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+PUSHD %1
+
+md build
+cd build
+
+call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64
+
+cmake.exe ..\llvm -G Ninja -DCMAKE_BUILD_TYPE=Release ^
+    -D LLVM_ENABLE_PROJECTS="clang;clang-tools-extra;libcxx;libcxxabi;lld" ^
+    -D LLVM_ENABLE_ASSERTIONS=ON ^
+    -DLLVM_LIT_ARGS="-v --xunit-xml-output test-results.xml"  
+
+POPD
\ No newline at end of file
diff --git a/scripts/run_ninja.bat b/scripts/run_ninja.bat
new file mode 100644
index 0000000..adb5094
--- /dev/null
+++ b/scripts/run_ninja.bat
@@ -0,0 +1,22 @@
+@rem Copyright 2019 Google LLC
+@rem
+@rem Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem     https://llvm.org/LICENSE.txt
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+PUSHD %1
+
+cd build
+
+call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64
+
+ninja %1
+POPD
\ No newline at end of file