From afed3a789803a90168f2a8452d36bb69856b8994 Mon Sep 17 00:00:00 2001 From: Mikhail Goncharov Date: Tue, 26 May 2020 12:53:19 +0200 Subject: [PATCH] Update docs and script for installing win machines I have tried to install new windows machine for buildkite and updated some scrips and docs on along the way. - Updated base image for k8s agent installation as it gave a warning that previous version was old. - Now buildkite secret is configured in a powershell script along with tags (and possible other parameters). - Split "windows_agent_start.ps1" to "..jenkins" and "..buildkite" as some parameters are different. - Created a "windows-development" machine in GCP stat can be resumed to build docker images / expriment. --- containers/agent-windows-buildkite/Dockerfile | 20 ++-- .../agent-windows-buildkite/start_agent.ps1 | 13 +- docs/playbooks.md | 104 ++++++++-------- kubernetes/windows_agent_create.sh | 2 +- scripts/windows_agent_bootstrap.ps1 | 111 +++++++++--------- scripts/windows_agent_start_buildkite.ps1 | 55 +++++++++ ...rt.ps1 => windows_agent_start_jenkins.ps1} | 110 +++++++++-------- 7 files changed, 227 insertions(+), 188 deletions(-) create mode 100644 scripts/windows_agent_start_buildkite.ps1 rename scripts/{windows_agent_start.ps1 => windows_agent_start_jenkins.ps1} (83%) diff --git a/containers/agent-windows-buildkite/Dockerfile b/containers/agent-windows-buildkite/Dockerfile index b79bb77..14714d6 100644 --- a/containers/agent-windows-buildkite/Dockerfile +++ b/containers/agent-windows-buildkite/Dockerfile @@ -1,14 +1,8 @@ -# escape=` - -# use windows server core image -ARG agent_windows_version -FROM gcr.io/llvm-premerge-checks/agent-windows:${agent_windows_version} - -# install buildkite agent as described in the documentation -# https://buildkite.com/docs/agent/v3/windows -# Supply your agent token via the arguement "-token + git config --global user.email + ``` 1. Install [Docker Enterprise](https://docs.docker.com/ee/docker-ee/windows/docker-ee/) and reboot: -```powershell -Install-Module DockerMsftProvider -Force -Install-Package Docker -ProviderName DockerMsftProvider -Force -Restart-Computer -``` -1. *optional:* install apps to help you work in the machine: -```powershell -choco install -y googlechrome vscode -``` -1. Log out of the machine and log back in. -1. Repeat until success: - 1. Start "Docker Desktop" and let it install it's dependencies. - Then reboot manually, when the error message pops up. - 1. If you have trouble with the machine name: try to shorten it to 16 chars. + ```powershell + Install-Module DockerMsftProvider -Force + Install-Package Docker -ProviderName DockerMsftProvider -Force + Restart-Computer + ``` 1. Configure the Docker credentials for GCP: -```powershell -gcloud components install docker-credential-gcr -docker-credential-gcr configure-docker -``` + ```powershell + gcloud init # set options according to ./k8s_config here + gcloud components install docker-credential-gcr + docker-credential-gcr configure-docker + ``` 1. To build and run the current agent run: -```powershell -git clone https://github.com/google/llvm-premerge-checks -cd llvm-premerge-checks\containers -powershell .\build_run.ps1 agent-windows-jenkins -``` -1. If you want to be able to push changes to github, you need to set up your github SSH keys and user name: -```powershell -ssh-keygen -git config --global user.name -git config --global user.email -``` - -To push push a new container run in `containers`: -```powershell -powershell .\build_deploy.ps1 -``` + ```powershell + cd c:\ + git clone https://github.com/google/llvm-premerge-checks + cd llvm-premerge-checks\containers + .\build_deploy.ps1 agent-windows-buildkite # or agent-windows-jenkins + c:\llvm-premerge-check\scripts\windows_agent_start_buildkite.ps1 # or windows_agent_start_jenkins.ps1 + ``` ## Spawning a new windows agent @@ -118,20 +113,21 @@ To spawn a new windows agent: 1. Go to the [GCP page](https://pantheon.corp.google.com/compute/instances?project=llvm-premerge-checks&instancessize=50) and pick a new number for the agent. 1. Run `kubernetes/windows_agent_create.sh agent-windows-` 1. Go to the [GCP page](https://pantheon.corp.google.com/compute/instances?project=llvm-premerge-checks&instancessize=50) again -1. login to the new machine via RDP (you probably need to set the i). -1. In the RDP session: run these commands in the CMD window to bootstrap the Windows machine: -```powershell -Invoke-WebRequest -uri 'https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/windows_agent_bootstrap.ps1' -OutFile windows_agent_bootstrap.ps1 -.\windows_agent_bootstrap.ps1 -``` -1. Ignore the pop-up to format the new disk. -1. Wait for the machine to reboot, then copy the credendials from `windows-agent-2` in `c:\credentials` to the new machine to `c:\credentials`. You can do that by mounting a folder of your host machine via RDP into the Windows VMs. -1: Copy the credientals to `C:\credentials` from another Windows agent. -1. Run this script to start the container: -```powershell -Invoke-WebRequest -uri 'https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/windows_agent_start.ps1' -OutFile windows_agent_start.ps1 -.\windows_agent_start.ps1 jenkins -``` +1. login to the new machine via RDP (you will need a RDP client, e.g. Chrome app). +1. In the RDP session: run these commands in the CMD window under Administrator to bootstrap the Windows machine: + ```powershell + Invoke-WebRequest -uri 'https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/windows_agent_bootstrap.ps1' -OutFile windows_agent_bootstrap.ps1 + ./windows_agent_bootstrap.ps1 + ``` + Ignore the pop-up to format the new disk andw wait for the machine to reboot. +1. Create `c:\credentials` folder with the agent credentials: + For *Buildkite* add file `buildkite-env.ps1`: + ```powershell + $Env:buildkiteAgentToken = "secret-token" + $Env:BUILDKITE_AGENT_TAGS = "queue=premerge;os=windows" + ``` + For *Jenkins*: `build-agent-results_key.json` to access cloud storage copy from one of the existing machines. +1. Start the container `C:\llvm-premerge-checks\scripts\windows_agent_start_[buildkite|jenkins].ps1 ` ## Testing scripts locally diff --git a/kubernetes/windows_agent_create.sh b/kubernetes/windows_agent_create.sh index cd98838..c7381ac 100755 --- a/kubernetes/windows_agent_create.sh +++ b/kubernetes/windows_agent_create.sh @@ -30,6 +30,6 @@ gcloud beta compute instances create "${NAME}" \ --zone="${GCP_ZONE}" \ --machine-type=n1-standard-16 \ --local-ssd=device-name=local-ssd-0 \ - --image=windows-server-2019-dc-v20191210 \ + --image=windows-server-2019-dc-for-containers-v20200512 \ --image-project=windows-cloud \ --boot-disk-size=100GB diff --git a/scripts/windows_agent_bootstrap.ps1 b/scripts/windows_agent_bootstrap.ps1 index 0723591..ae93b46 100644 --- a/scripts/windows_agent_bootstrap.ps1 +++ b/scripts/windows_agent_bootstrap.ps1 @@ -1,54 +1,57 @@ -# Copyright 2019 Google LLC - -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# https://llvm.org/LICENSE.txt - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# 1st stage of the installation process. -# This script only needs to be run once per machine. - -Write-Host "Initializing local SSD..." -New-Variable -Name diskid -Value (Get-Disk -FriendlyName "Google EphemeralDisk").Number -#New-Variable -Name diskid -Value (Get-Disk -FriendlyName "NVMe nvme_card").Number - -# TODO: check if machine has an SSD -# TODO: only do this, if SSD is not yet partioned and formatted -Initialize-Disk -Number $diskid -New-Partition -DiskNumber $diskid -UseMaximumSize -AssignDriveLetter -Format-Volume -DriveLetter D - -Write-Host "install chocolately as package manager..." -iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')) -choco feature disable --name showDownloadProgress -choco install -y git - -# move docker folder to SSD to get better IO performance -New-Item -Path "D:\" -Name "Docker" -ItemType "directory" -cmd /C "mklink /j C:\ProgramData\Docker D:\docker" - -# create folder for credentials -New-Item -Path "C:\" -Name "credentials" -ItemType "directory" - -# install Docker -Install-PackageProvider -Name NuGet -Force -Install-Module -Name DockerMsftProvider -Repository PSGallery -Force -Install-Package -Name docker -ProviderName DockerMsftProvider -Force -sc.exe config docker start=delayed-auto - -# install gcloud and authenticate access to gcr.io registry -# TODO: find a better way to install the Google Cloud SDK, avoid ingoring the checksum -choco install -y gcloudsdk --ignore-checksums - -# exclude drive d from Virus scans, to get better performance -Add-MpPreference -ExclusionPath “D:\” - -# Reboot -Restart-Computer -Force +# Copyright 2019 Google LLC + +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://llvm.org/LICENSE.txt + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 1st stage of the installation process. +# This script only needs to be run once per machine. + +Write-Host "Initializing local SSD..." +New-Variable -Name diskid -Value (Get-Disk -FriendlyName "Google EphemeralDisk").Number +#New-Variable -Name diskid -Value (Get-Disk -FriendlyName "NVMe nvme_card").Number + +# TODO: check if machine has an SSD +# TODO: only do this, if SSD is not yet partioned and formatted +Initialize-Disk -Number $diskid +New-Partition -DiskNumber $diskid -UseMaximumSize -AssignDriveLetter +Format-Volume -DriveLetter D + +Write-Host "install chocolately as package manager..." +iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')) +choco feature disable --name showDownloadProgress +choco install -y git + +# move docker folder to SSD to get better IO performance +New-Item -Path "D:\" -Name "Docker" -ItemType "directory" +cmd /C "mklink /j C:\ProgramData\Docker D:\docker" + +# create folder for credentials +New-Item -Path "C:\" -Name "credentials" -ItemType "directory" + +# install Docker +Install-PackageProvider -Name NuGet -Force +Install-Module -Name DockerMsftProvider -Repository PSGallery -Force +Install-Package -Name docker -ProviderName DockerMsftProvider -Force +sc.exe config docker start=delayed-auto + +# install gcloud and authenticate access to gcr.io registry +# TODO: find a better way to install the Google Cloud SDK, avoid ingoring the checksum +choco install -y gcloudsdk --ignore-checksums + +# exclude drive d from Virus scans, to get better performance +Add-MpPreference -ExclusionPath “D:\” + +# clone scripts repo (this one) +git clone https://github.com/google/llvm-premerge-checks.git "c:\llvm-premerge-checks" + +# Reboot +Restart-Computer -Force diff --git a/scripts/windows_agent_start_buildkite.ps1 b/scripts/windows_agent_start_buildkite.ps1 new file mode 100644 index 0000000..f242968 --- /dev/null +++ b/scripts/windows_agent_start_buildkite.ps1 @@ -0,0 +1,55 @@ +# Copyright 2019 Google LLC + +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://llvm.org/LICENSE.txt + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Pull and start the Docker container for a Windows agent. +# To setup a Windows agent see docs/playbooks.md + +param( + [string]$version = "latest", + [switch]$testing = $false +) + +$NAME="agent-windows-buildkite" +$IMAGE="gcr.io/llvm-premerge-checks/${NAME}:${version}" + +Write-Output "Authenticating docker..." +Write-Output "y`n" | gcloud auth configure-docker + +Write-Output "Pulling new image..." +docker pull ${IMAGE} + +Write-Output "Stopping old container..." +docker stop ${NAME} +docker rm ${NAME} + +Write-Output "Starting container..." +if (${testing}) { + docker run -it ` + -v D:\:C:\ws ` + -v C:\credentials:C:\credentials ` + -e BUILDKITE_AGENT_NAME=$env:computername ` + -e BUILDKITE_BUILD_PATH=C:\ws ` + --restart unless-stopped ` + --name ${NAME} ` + ${IMAGE} powershell +} else { + docker run -d ` + -v D:\:C:\ws ` + -v C:\credentials:C:\credentials ` + -e BUILDKITE_AGENT_NAME=$env:computername ` + -e BUILDKITE_BUILD_PATH=C:\ws ` + --restart unless-stopped ` + --name ${NAME} ` + ${IMAGE} +} diff --git a/scripts/windows_agent_start.ps1 b/scripts/windows_agent_start_jenkins.ps1 similarity index 83% rename from scripts/windows_agent_start.ps1 rename to scripts/windows_agent_start_jenkins.ps1 index bce8315..69b4e90 100644 --- a/scripts/windows_agent_start.ps1 +++ b/scripts/windows_agent_start_jenkins.ps1 @@ -1,57 +1,53 @@ -# Copyright 2019 Google LLC - -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# https://llvm.org/LICENSE.txt - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Pull and start the Docker container for a Windows agent. -# To setup a Windows agent see docs/playbooks.md - -# TODO: add parameter to bootstrap buildkite or jenkins - -param( - [Parameter(Mandatory=$true)] - [ValidateSet("buildkite", "jenkins")] - [string]$master, - [switch]$testing = $false -) - -$NAME="agent-windows-${master}" -$IMAGE="gcr.io/llvm-premerge-checks/${NAME}" - -Write-Output "Authenticating docker..." -Write-Output "y`n" | gcloud auth configure-docker - -Write-Output "Pulling new image..." -docker pull ${IMAGE} - -Write-Output "Stopping old container..." -docker stop ${NAME} -docker rm ${NAME} - -Write-Output "Starting container..." -if (${testing}) { - docker run -it ` - -v D:\:C:\ws ` - -v C:\credentials:C:\credentials ` - -e PARENT_HOSTNAME=$env:computername ` - --restart unless-stopped ` - --name ${NAME} ` - ${IMAGE} powershell -} else { - docker run -d ` - -v D:\:C:\ws ` - -v C:\credentials:C:\credentials ` - -e PARENT_HOSTNAME=$env:computername ` - --restart unless-stopped ` - --name ${NAME} ` - ${IMAGE} -} +# Copyright 2019 Google LLC + +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://llvm.org/LICENSE.txt + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Pull and start the Docker container for a Windows agent. +# To setup a Windows agent see docs/playbooks.md + +param( + [string]$version = "latest", + [switch]$testing = $false +) + +$NAME="agent-windows-jenkins" +$IMAGE="gcr.io/llvm-premerge-checks/${NAME}:${version}" + +Write-Output "Authenticating docker..." +Write-Output "y`n" | gcloud auth configure-docker + +Write-Output "Pulling new image..." +docker pull ${IMAGE} + +Write-Output "Stopping old container..." +docker stop ${NAME} +docker rm ${NAME} + +Write-Output "Starting container..." +if (${testing}) { + docker run -it ` + -v D:\:C:\ws ` + -v C:\credentials:C:\credentials ` + -e PARENT_HOSTNAME=$env:computername ` + --restart unless-stopped ` + --name ${NAME} ` + ${IMAGE} powershell +} else { + docker run -d ` + -v D:\:C:\ws ` + -v C:\credentials:C:\credentials ` + -e PARENT_HOSTNAME=$env:computername ` + --restart unless-stopped ` + --name ${NAME} ` + ${IMAGE} +}