revi-archive/llvm-premerge-checks
1
0
Fork 0
Code Activity

remove jenkins configs and scrips

Browse source 
- moved linux agents to a new node pool (basically renamed old one)

- removed some out of date scripts or moved them to playlists

- removed shell scrips invoked from jenkins

- minor docs updates

Some python classes methods are not used anymore and will be cleaned up
separately.
This commit is contained in:
Mikhail Goncharov 2020-08-27 14:44:56 +02:00
parent 5380ea63b4
commit 8b36ff5888
70 changed files with 31 additions and 3444 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

98
Jenkins/BETA-apply-patch/Jenkinsfile vendored
View file

@ -1,98 +0,0 @@
// Copyright 2019 Google LLC
//
// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://llvm.org/LICENSE.txt
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
def success = true
def failure_message = ""
pipeline {
agent { label 'linux' }
parameters {
string(name: 'DIFF_ID')
string(name: 'PHID')
string(name: 'REV_ID')
}
environment {
CONDUIT_TOKEN = credentials('phabricator-conduit-token')
PHABRICATOR_HOST = 'https://reviews.llvm.org'
LLVM_DIR = "${WORKSPACE}/llvm-project"
SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts"
RESULT_DIR = "${WORKSPACE}/results"
PHAB_LOG = "${RESULT_DIR}/.phabricator-comment"
MY_BUILD_ID = "${JOB_BASE_NAME}-${BUILD_NUMBER}"
TARGET_DIR = "/mnt/nfs/results/${MY_BUILD_ID}"
RESULT_URL = "http://results.llvm-merge-guard.org/${MY_BUILD_ID}"
}
options {
timeout(time:10, unit:'MINUTES')
}
stages {
stage("build info"){
steps {
echo "Building diff ${DIFF_ID} with PHID ${PHID} for Revision ${REV_ID}"
script {
currentBuild.displayName += " D${REV_ID}"
currentBuild.description = "<a href='https://reviews.llvm.org/D${REV_ID}'>D${REV_ID}</a>"
}
}
}
stage("git checkout"){
steps {
dir("${LLVM_DIR}"){
git url: 'git@github.com:llvm-premerge-tests/llvm-project.git'
}
dir("llvm-premerge-checks")
{
git url: 'https://github.com/google/llvm-premerge-checks.git'
}
sh """
rm -rf ${RESULT_DIR}
mkdir -p ${RESULT_DIR}
"""
}
}
stage('arc patch'){
steps {
dir("${LLVM_DIR}"){
sh """
${SCRIPT_DIR}/phabtalk/apply_patch2.py ${DIFF_ID} \
--token ${CONDUIT_TOKEN} \
--url ${PHABRICATOR_HOST} \
--push-branch
"""
}
}
}
}
post {
always {
dir("${RESULT_DIR}") {
// copy console log to result folder
sh "wget -qO console-log.txt http://jenkins-ui.jenkins.svc.cluster.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText"
sh "mkdir ${TARGET_DIR}"
sh "cp * ${TARGET_DIR}"
}
sh """${SCRIPT_DIR}/phabtalk/phabtalk.py "${PHID}" "${DIFF_ID}" \
--workspace "${LLVM_DIR}" \
--conduit-token "${CONDUIT_TOKEN}" \
--host "${PHABRICATOR_HOST}/api/" \
--buildresult ${currentBuild.result} \
--results-dir "${TARGET_DIR}" \
--results-url "${RESULT_URL}" \
--failures "Applying patch failed" \
--name "apply patch"
"""
}
}
}

184
Jenkins/BETA-phabricator-linux/Jenkinsfile vendored
View file

@ -1,184 +0,0 @@
// Copyright 2019 Google LLC
//
// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://llvm.org/LICENSE.txt
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
def success = true
def failure_message = ""
pipeline {
agent { label 'linux' }
parameters {
string(name: 'DIFF_ID')
string(name: 'PHID')
string(name: 'REV_ID')
}
environment {
CONDUIT_TOKEN = credentials('phabricator-conduit-token')
PHABRICATOR_HOST = 'https://reviews.llvm.org'
LLVM_DIR = "${WORKSPACE}/llvm-project"
SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts"
RESULT_DIR = "${WORKSPACE}/results"
PHAB_LOG = "${RESULT_DIR}/.phabricator-comment"
MY_BUILD_ID = "${JOB_BASE_NAME}-${BUILD_NUMBER}"
TARGET_DIR = "/mnt/nfs/results/${MY_BUILD_ID}"
RESULT_URL = "http://results.llvm-merge-guard.org/${MY_BUILD_ID}"
}
options {
timeout(time:2, unit:'HOURS')
}
stages {
stage("build info"){
steps {
echo "Building diff ${DIFF_ID} with PHID ${PHID} for Revision ${REV_ID}"
script {
currentBuild.displayName += " D${REV_ID}"
currentBuild.description = "<a href='https://reviews.llvm.org/D${REV_ID}'>D${REV_ID}</a>"
}
sh """
rm -rf ${RESULT_DIR}
mkdir -p ${RESULT_DIR}
"""
// Report versions of the installed packages.
sh """
echo Versions of various tools: > ${RESULT_DIR}/package_version.log
clang --version >> ${RESULT_DIR}/package_version.log
clang-tidy --version >> ${RESULT_DIR}/package_version.log
ld.lld --version >> ${RESULT_DIR}/package_version.log
dpkg -l >> ${RESULT_DIR}/package_version.log
"""
}
}
stage("git checkout")
{
steps {
dir("${LLVM_DIR}")
{
git url: 'git@github.com:llvm-premerge-tests/llvm-project.git', branch: "phab-diff-${DIFF_ID}"
sh 'git clean -fdx'
sh 'git show -s'
}
dir("llvm-premerge-checks")
{
git url: 'https://github.com/google/llvm-premerge-checks.git'
}
}
}
stage('CMake') {
steps {
dir("${LLVM_DIR}")
{
script {
success = false
failure_message = "Failed to run cmake"
}
sh "${SCRIPT_DIR}/run_cmake.py detect"
script {
success = true
failure_message = ""
}
}
}
}
stage('ninja all') {
steps {
dir("${LLVM_DIR}")
{
script {
try {
sh(script: "${SCRIPT_DIR}/run_ninja.py all")
} catch (e) {
success = false;
failure_message = "'ninja all' failed" // append as build might already be broken
echo e.toString()
}
}
}
}
}
stage('ninja check-all') {
steps {
dir("${LLVM_DIR}")
{
script {
if (success) {
// only run tests if build has passed, see #176
try {
sh(script: "${SCRIPT_DIR}/run_ninja.py check-all")
} catch (e) {
success = false;
failure_message += "\n'ninja check-all' failed" // append as build might already be broken
echo e.toString()
}
}
}
}
}
}
stage('linters') {
steps {
dir("${LLVM_DIR}")
{
script {
try {
sh(script: "${SCRIPT_DIR}/lint.sh HEAD~1 ${RESULT_DIR}")
} catch (e) {
success = false;
failure_message += "\nFailed to run linters" // append as build might already be broken
echo e.toString()
}
}
}
}
}
}
post {
always {
script {
if (success) {
currentBuild.result = "SUCCESS"
} else {
currentBuild.result = "FAILURE"
}
}
echo "Console log is available at ${RESULT_URL}"
dir("${RESULT_DIR}") {
// copy console log to result folder
sh "wget -qO console-log.txt http://jenkins-ui.jenkins.svc.cluster.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText"
// keep a copy of the answer from Phabricator for debugging
// TODO: move all file copy operations here
sh """
set -eu
cp ${LLVM_DIR}/build/CMakeCache.txt . || :
cp ${LLVM_DIR}/build/test-results.xml . || :
mkdir -p ${TARGET_DIR}
cp * ${TARGET_DIR}
"""
}
/// send results to Phabricator
sh """${SCRIPT_DIR}/phabtalk/phabtalk.py "${PHID}" "${DIFF_ID}" \
--workspace "${WORKSPACE}" \
--conduit-token "${CONDUIT_TOKEN}" \
--test-result-file "${RESULT_DIR}/test-results.xml" \
--host "${PHABRICATOR_HOST}/api/" \
--buildresult ${currentBuild.result} \
--clang-format-patch "${RESULT_DIR}/clang-format.patch" \
--clang-tidy-result "${RESULT_DIR}/clang-tidy.txt" \
--clang-tidy-ignore "${SCRIPT_DIR}/clang-tidy-comments.ignore" \
--results-dir "${TARGET_DIR}" \
--results-url "${RESULT_URL}" \
--failures "${failure_message}" \
--name "linux"
"""
}
}
}

177
Jenkins/BETA-phabricator-windows/Jenkinsfile vendored
View file

@ -1,177 +0,0 @@
// Copyright 2019 Google LLC
//
// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://llvm.org/LICENSE.txt
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
def success = true
pipeline {
agent {
node {
label 'windows'
// use custom workspace folder to shorten paths
customWorkspace "C:\\ws\\beta"
}
}
parameters {
string(name: 'DIFF_ID')
string(name: 'PHID')
string(name: 'REV_ID')
}
environment {
CONDUIT_TOKEN = credentials('phabricator-conduit-token')
PHABRICATOR_HOST = 'https://reviews.llvm.org'
PHAB_LOG = "${WORKSPACE}/build/.phabricator-comment"
MY_BUILD_ID = "${JOB_BASE_NAME}-${BUILD_NUMBER}"
RESULT_URL = "https://storage.googleapis.com/llvm-premerge-checks/results/${MY_BUILD_ID}"
SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts"
// store all build results here, will be uploaded to GCS later
RESULT_DIR = "${WORKSPACE}\\results"
LLVM_DIR = "${WORKSPACE}\\llvm-project"
// enable sccache for this build. Comment out the line to disable it
SCCACHE_DIR = "C:\\ws\\sccache"
}
options {
timeout(time:2, unit:'HOURS')
}
stages {
stage("build info"){
steps {
echo "Building diff ${DIFF_ID} with PHID ${PHID} for Revision ${REV_ID}"
script {
currentBuild.displayName += " D${REV_ID}"
currentBuild.description = "<a href='https://reviews.llvm.org/D${REV_ID}'>D${REV_ID}</a>"
}
script {
success = true
failure_message = ""
}
}
}
stage("git checkout"){
steps {
echo "getting llvm-premerge-checks... "
dir("llvm-premerge-checks")
{
git url: 'https://github.com/google/llvm-premerge-checks.git'
}
echo "getting llvm-project... "
dir("llvm-project")
{
git url: 'https://github.com/llvm-premerge-tests/llvm-project.git', branch: "phab-diff-${DIFF_ID}"
powershell 'git clean -fdx'
powershell 'git show -s'
}
// create ${RESULT_DIR}
powershell """
Remove-Item ${RESULT_DIR} -Recurse -ErrorAction Ignore
New-Item -ItemType Directory -Force -Path ${RESULT_DIR} | Out-Null
"""
}
}
stage('CMake') {
steps {
dir("${LLVM_DIR}"){
powershell "python ${SCRIPT_DIR}/run_cmake.py detect"
}
}
post {
failure {
script {
success = false
failure_message = "Failed to run CMake"
}
}
}
}
stage('ninja all') {
steps {
dir("${LLVM_DIR}"){
powershell "python ${SCRIPT_DIR}/run_ninja.py all"
}
}
post {
failure {
script {
success = false
failure_message = "Failed to run ninja all"
}
}
}
}
stage('ninja check-all') {
steps {
dir("${LLVM_DIR}"){
powershell "python ${SCRIPT_DIR}/run_ninja.py check-all"
}
}
post {
failure {
script {
success = false
failure_message = "Failed to run ninja check-all"
}
}
}
}
}
post {
always {
script {
if (success) {
currentBuild.result = "SUCCESS"
} else {
currentBuild.result = "FAILURE"
}
}
echo "Uploading logs to ${RESULT_URL} ..."
dir("${RESULT_DIR}") {
// gather all result files in a folder
powershell """
\$ErrorActionPreference = 'Continue'
Write-Host "Getting the console log..."
Invoke-WebRequest -OutFile console-log.txt -uri "http://jenkins.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText" -ErrorAction "Continue"
Write-Host "Copying build artefacts..."
Copy-Item "${LLVM_DIR}\\choose_projects.log"
Copy-Item "${LLVM_DIR}\\build\\CMakeCache.txt"
Copy-Item "${LLVM_DIR}\\build\\test-results.xml"
"""
}
/// send results to Phabricator
echo "Sending build feedback to Phabricator..."
bat """
python ${SCRIPT_DIR}/phabtalk/phabtalk.py "${PHID}" "${DIFF_ID}" ^
--workspace "${LLVM_DIR}" ^
--conduit-token "${CONDUIT_TOKEN}" ^
--test-result-file "test-results.xml" ^
--host "${PHABRICATOR_HOST}/api/" ^
--results-dir "${RESULT_DIR}" ^
--results-url "${RESULT_URL}" ^
--failures "${failure_message}" ^
--buildresult ${currentBuild.result} ^
--name "windows"
"""
dir("${RESULT_DIR}") {
// upload results to
// Google Cloud Storage
powershell """
Write-Host "Uploading results to GCS..."
\$ErrorActionPreference = 'Continue'
gsutil cp *.* gs://llvm-premerge-checks/results/${MY_BUILD_ID}/
Write-Host "Done."
"""
}
echo "Done."
}
}
}

98
Jenkins/apply-patch/Jenkinsfile vendored
View file

@ -1,98 +0,0 @@
// Copyright 2019 Google LLC
//
// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://llvm.org/LICENSE.txt
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
def success = true
def failure_message = ""
pipeline {
agent { label 'linux' }
parameters {
string(name: 'DIFF_ID')
string(name: 'PHID')
string(name: 'REV_ID')
}
environment {
CONDUIT_TOKEN = credentials('phabricator-conduit-token')
PHABRICATOR_HOST = 'https://reviews.llvm.org'
LLVM_DIR = "${WORKSPACE}/llvm-project"
SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts"
RESULT_DIR = "${WORKSPACE}/results"
PHAB_LOG = "${RESULT_DIR}/.phabricator-comment"
MY_BUILD_ID = "${JOB_BASE_NAME}-${BUILD_NUMBER}"
TARGET_DIR = "/mnt/nfs/results/${MY_BUILD_ID}"
RESULT_URL = "http://results.llvm-merge-guard.org/${MY_BUILD_ID}"
}
options {
timeout(time:10, unit:'MINUTES')
}
stages {
stage("build info"){
steps {
echo "Building diff ${DIFF_ID} with PHID ${PHID} for Revision ${REV_ID}"
script {
currentBuild.displayName += " D${REV_ID}"
currentBuild.description = "<a href='https://reviews.llvm.org/D${REV_ID}'>D${REV_ID}</a>"
}
}
}
stage("git checkout"){
steps {
dir("${LLVM_DIR}"){
git url: 'git@github.com:llvm-premerge-tests/llvm-project.git'
}
dir("llvm-premerge-checks")
{
git url: 'https://github.com/google/llvm-premerge-checks.git'
}
sh """
rm -rf ${RESULT_DIR}
mkdir -p ${RESULT_DIR}
"""
}
}
stage('arc patch'){
steps {
dir("${LLVM_DIR}"){
sh """
${SCRIPT_DIR}/phabtalk/apply_patch2.py ${DIFF_ID} \
--token ${CONDUIT_TOKEN} \
--url ${PHABRICATOR_HOST} \
--push-branch
"""
}
}
}
}
post {
always {
dir("${RESULT_DIR}") {
// copy console log to result folder
sh "wget -qO console-log.txt http://jenkins-ui.jenkins.svc.cluster.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText"
sh "mkdir ${TARGET_DIR}"
sh "cp * ${TARGET_DIR}"
}
sh """${SCRIPT_DIR}/phabtalk/phabtalk.py "${PHID}" "${DIFF_ID}" \
--workspace "${LLVM_DIR}" \
--conduit-token "${CONDUIT_TOKEN}" \
--host "${PHABRICATOR_HOST}/api/" \
--buildresult ${currentBuild.result} \
--results-dir "${TARGET_DIR}" \
--results-url "${RESULT_URL}" \
--failures "Applying patch failed" \
--name "apply patch"
"""
}
}
}

56
Jenkins/daily-cleanup/Jenkinsfile vendored
View file

@ -1,56 +0,0 @@
// Copyright 2019 Google LLC
//
// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://llvm.org/LICENSE.txt
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/* Pipeline executed daily to do cleanup work */
def success = true
def failure_message = ""
pipeline {
agent { label 'linux' }
triggers {
cron('@daily')
}
environment {
LLVM_DIR = "${WORKSPACE}/llvm-project"
SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts"
}
options {
timeout(time:30, unit:'MINUTES')
}
stages {
stage("git checkout"){
steps {
dir("${LLVM_DIR}"){
git url: 'git@github.com:llvm-premerge-tests/llvm-project.git'
}
dir("llvm-premerge-checks")
{
git url: 'https://github.com/google/llvm-premerge-checks.git'
}
}
}
stage('cleanup repo'){
// delete old branches after 30 days
steps {
dir("${LLVM_DIR}"){
sh """
python ${SCRIPT_DIR}/cleanup_branches.py --days 30 --pattern "phab-diff-.*"
"""
}
}
}
}
}

71
Jenkins/master-linux-pipeline/Jenkinsfile vendored
View file

@ -1,71 +0,0 @@
// Copyright 2019 Google LLC
//
// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://llvm.org/LICENSE.txt
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pipeline {
agent { label 'linux' }
triggers {
pollSCM 'H H/2 * * *'
}
environment {
BUILD_ID="${JOB_BASE_NAME}-${BUILD_NUMBER}"
TARGET_DIR="/mnt/nfs/results/${BUILD_ID}"
SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts"
}
options {
// set build timeout
timeout(time:2, unit:'HOURS')
}
stages {
stage("git checkout"){
steps {
git url: 'https://github.com/llvm/llvm-project.git'
sh 'git clean -fdx'
sh 'mkdir -p llvm-premerge-checks'
dir("llvm-premerge-checks")
{
git url: 'https://github.com/google/llvm-premerge-checks.git'
}
sh 'rm -rf build || true'
sh 'mkdir -p build'
sh 'mkdir -p "${TARGET_DIR}"'
}
}
stage('CMake') {
steps {
sh "${SCRIPT_DIR}/run_cmake.py"
}
}
stage('ninja all') {
steps {
sh "${SCRIPT_DIR}/run_ninja.py all"
}
}
stage('ninja check-all') {
steps {
sh "${SCRIPT_DIR}/run_ninja.py check-all"
}
}
}
post {
always {
echo "Console log is available at https://results.llvm-merge-guard.org/${BUILD_ID}"
dir("${env.TARGET_DIR}") {
// copy console log to result folder
sh "wget -qO console-log.txt http://jenkins-ui.jenkins.svc.cluster.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText"
// TODO: move all file copy operations here
// junit 'build/*.xml'
}
}
}
}

106
Jenkins/master-windows-pipeline/Jenkinsfile vendored
View file

@ -1,106 +0,0 @@
// Copyright 2019 Google LLC
//
// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://llvm.org/LICENSE.txt
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pipeline {
agent {
node {
label 'windows'
// use custom workspace folder to shorten paths
customWorkspace "C:\\ws\\master"
}
}
triggers {
pollSCM 'H H/4 * * *'
}
options {
// enable timestaps for getting execution times
timestamps ()
// set build timeout
timeout(time:2, unit:'HOURS')
}
environment {
MY_BUILD_ID = "${JOB_BASE_NAME}-${BUILD_NUMBER}"
SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts"
RESULT_DIR = "${WORKSPACE}\\results"
LLVM_DIR = "${WORKSPACE}\\llvm-project"
// enable sccache for this build. Comment out the line to disable it
SCCACHE_DIR = "C:\\ws\\sccache"
RESULT_URL = "https://storage.googleapis.com/llvm-premerge-checks/results/${MY_BUILD_ID}"
}
stages {
stage("git checkout"){
steps {
echo "getting llvm-premerge-checks... "
dir("llvm-premerge-checks")
{
git url: 'https://github.com/google/llvm-premerge-checks.git'
}
echo "getting llvm-project... "
dir("llvm-project")
{
git url: 'https://github.com/llvm/llvm-project.git'
}
powershell "New-Item -ItemType Directory -Force -Path ${RESULT_DIR}"
}
}
stage('CMake') {
steps {
dir("${LLVM_DIR}"){
powershell "python ${SCRIPT_DIR}/run_cmake.py"
}
}
}
stage('ninja all') {
steps {
dir("${LLVM_DIR}"){
powershell "python ${SCRIPT_DIR}/run_ninja.py all"
}
}
}
stage('ninja check-all') {
steps {
dir("${LLVM_DIR}"){
powershell "python ${SCRIPT_DIR}/run_ninja.py check-all"
}
}
}
}
post {
always {
echo "Logs uploaded to ${RESUlT_URL}"
dir("${env.RESULT_DIR}") {
// gather all result files in a folder, then upload everything to
// Google Cloud Storage
powershell """
# get the console log
Invoke-WebRequest -OutFile console-log.txt -uri "http://jenkins.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText" -ErrorAction "Continue"
Copy-Item "${LLVM_DIR}\\build\\CMakeCache.txt"
Copy-Item "${LLVM_DIR}\\build\\test-results.xml"
# upload files
\$ErrorActionPreference = 'SilentlyContinue'
gsutil cp -Z *.* gs://llvm-premerge-checks/results/${MY_BUILD_ID}/
"""
}
// doesn't find junit results, not sure why...
// junit "${LLVM_DIR}\\build\\test-results.xml"
}
}
}

181
Jenkins/phabricator-linux/Jenkinsfile vendored
View file

@ -1,181 +0,0 @@
// Copyright 2019 Google LLC
//
// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://llvm.org/LICENSE.txt
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
def success = true
def failure_message = ""
pipeline {
agent { label 'linux' }
parameters {
string(name: 'DIFF_ID')
string(name: 'PHID')
string(name: 'REV_ID')
}
environment {
CONDUIT_TOKEN = credentials('phabricator-conduit-token')
PHABRICATOR_HOST = 'https://reviews.llvm.org'
LLVM_DIR = "${WORKSPACE}/llvm-project"
SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts"
RESULT_DIR = "${WORKSPACE}/results"
PHAB_LOG = "${RESULT_DIR}/.phabricator-comment"
MY_BUILD_ID = "${JOB_BASE_NAME}-${BUILD_NUMBER}"
TARGET_DIR = "/mnt/nfs/results/${MY_BUILD_ID}"
RESULT_URL = "http://results.llvm-merge-guard.org/${MY_BUILD_ID}"
}
options {
timeout(time:2, unit:'HOURS')
}
stages {
stage("build info"){
steps {
echo "Building diff ${DIFF_ID} with PHID ${PHID} for Revision ${REV_ID}"
script {
currentBuild.displayName += " D${REV_ID}"
currentBuild.description = "<a href='https://reviews.llvm.org/D${REV_ID}'>D${REV_ID}</a>"
}
sh """
rm -rf ${RESULT_DIR}
mkdir -p ${RESULT_DIR}
"""
// Report versions of the installed packages.
sh """
echo Versions of various tools: > ${RESULT_DIR}/package_version.log
clang --version >> ${RESULT_DIR}/package_version.log
clang-tidy --version >> ${RESULT_DIR}/package_version.log
ld.lld --version >> ${RESULT_DIR}/package_version.log
dpkg -l >> ${RESULT_DIR}/package_version.log
"""
}
}
stage("git checkout")
{
steps {
dir("${LLVM_DIR}")
{
git url: 'git@github.com:llvm-premerge-tests/llvm-project.git', branch: "phab-diff-${DIFF_ID}"
sh 'git clean -fdx'
sh 'git show -s'
}
dir("llvm-premerge-checks")
{
git url: 'https://github.com/google/llvm-premerge-checks.git'
}
}
}
stage('CMake') {
steps {
dir("${LLVM_DIR}")
{
script {
success = false
failure_message = "Failed to run cmake"
}
sh "${SCRIPT_DIR}/run_cmake.py detect"
script {
success = true
failure_message = ""
}
}
}
}
stage('ninja all') {
steps {
dir("${LLVM_DIR}")
{
script {
try {
sh(script: "${SCRIPT_DIR}/run_ninja.py all")
} catch (e) {
success = false;
failure_message = "'ninja all' failed" // append as build might already be broken
echo e.toString()
}
}
}
}
}
stage('ninja check-all') {
steps {
dir("${LLVM_DIR}")
{
script {
try {
sh(script: "${SCRIPT_DIR}/run_ninja.py check-all")
} catch (e) {
success = false;
failure_message += "\n'ninja check-all' failed" // append as build might already be broken
echo e.toString()
}
}
}
}
}
stage('linters') {
steps {
dir("${LLVM_DIR}")
{
script {
try {
sh(script: "${SCRIPT_DIR}/lint.sh HEAD~1 ${RESULT_DIR}")
} catch (e) {
success = false;
failure_message += "\nFailed to run linters" // append as build might already be broken
echo e.toString()
}
}
}
}
}
}
post {
always {
script {
if (success) {
currentBuild.result = "SUCCESS"
} else {
currentBuild.result = "FAILURE"
}
}
echo "Console log is available at ${RESULT_URL}"
dir("${RESULT_DIR}") {
// copy console log to result folder
sh "wget -qO console-log.txt http://jenkins-ui.jenkins.svc.cluster.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText"
// keep a copy of the answer from Phabricator for debugging
// TODO: move all file copy operations here
sh """
set -eu
cp ${LLVM_DIR}/build/CMakeCache.txt . || :
cp ${LLVM_DIR}/build/test-results.xml . || :
mkdir -p ${TARGET_DIR}
cp * ${TARGET_DIR}
"""
}
/// send results to Phabricator
sh """${SCRIPT_DIR}/phabtalk/phabtalk.py "${PHID}" "${DIFF_ID}" \
--workspace "${WORKSPACE}" \
--conduit-token "${CONDUIT_TOKEN}" \
--test-result-file "${RESULT_DIR}/test-results.xml" \
--host "${PHABRICATOR_HOST}/api/" \
--buildresult ${currentBuild.result} \
--clang-format-patch "${RESULT_DIR}/clang-format.patch" \
--clang-tidy-result "${RESULT_DIR}/clang-tidy.txt" \
--clang-tidy-ignore "${SCRIPT_DIR}/clang-tidy-comments.ignore" \
--results-dir "${TARGET_DIR}" \
--results-url "${RESULT_URL}" \
--failures "${failure_message}" \
--name "linux"
"""
}
}
}

177
Jenkins/phabricator-windows/Jenkinsfile vendored
View file

@ -1,177 +0,0 @@
// Copyright 2019 Google LLC
//
// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://llvm.org/LICENSE.txt
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
def success = true
pipeline {
agent {
node {
label 'windows'
// use custom workspace folder to shorten paths
customWorkspace "C:\\ws\\prod"
}
}
parameters {
string(name: 'DIFF_ID')
string(name: 'PHID')
string(name: 'REV_ID')
}
environment {
CONDUIT_TOKEN = credentials('phabricator-conduit-token')
PHABRICATOR_HOST = 'https://reviews.llvm.org'
PHAB_LOG = "${WORKSPACE}/build/.phabricator-comment"
MY_BUILD_ID = "${JOB_BASE_NAME}-${BUILD_NUMBER}"
RESULT_URL = "https://storage.googleapis.com/llvm-premerge-checks/results/${MY_BUILD_ID}"
SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts"
// store all build results here, will be uploaded to GCS later
RESULT_DIR = "${WORKSPACE}\\results"
LLVM_DIR = "${WORKSPACE}\\llvm-project"
// enable sccache for this build. Comment out the line to disable it
SCCACHE_DIR = "C:\\ws\\sccache"
}
options {
timeout(time:2, unit:'HOURS')
}
stages {
stage("build info"){
steps {
echo "Building diff ${DIFF_ID} with PHID ${PHID} for Revision ${REV_ID}"
script {
currentBuild.displayName += " D${REV_ID}"
currentBuild.description = "<a href='https://reviews.llvm.org/D${REV_ID}'>D${REV_ID}</a>"
}
script {
success = true
failure_message = ""
}
}
}
stage("git checkout"){
steps {
echo "getting llvm-premerge-checks... "
dir("llvm-premerge-checks")
{
git url: 'https://github.com/google/llvm-premerge-checks.git'
}
echo "getting llvm-project... "
dir("llvm-project")
{
git url: 'https://github.com/llvm-premerge-tests/llvm-project.git', branch: "phab-diff-${DIFF_ID}"
powershell 'git clean -fdx'
powershell 'git show -s'
}
// create ${RESULT_DIR}
powershell """
Remove-Item ${RESULT_DIR} -Recurse -ErrorAction Ignore
New-Item -ItemType Directory -Force -Path ${RESULT_DIR} | Out-Null
"""
}
}
stage('CMake') {
steps {
dir("${LLVM_DIR}"){
powershell "python ${SCRIPT_DIR}/run_cmake.py detect"
}
}
post {
failure {
script {
success = false
failure_message = "Failed to run CMake"
}
}
}
}
stage('ninja all') {
steps {
dir("${LLVM_DIR}"){
powershell "python ${SCRIPT_DIR}/run_ninja.py all"
}
}
post {
failure {
script {
success = false
failure_message = "Failed to run ninja all"
}
}
}
}
stage('ninja check-all') {
steps {
dir("${LLVM_DIR}"){
powershell "python ${SCRIPT_DIR}/run_ninja.py check-all"
}
}
post {
failure {
script {
success = false
failure_message = "Failed to run ninja check-all"
}
}
}
}
}
post {
always {
script {
if (success) {
currentBuild.result = "SUCCESS"
} else {
currentBuild.result = "FAILURE"
}
}
echo "Uploading logs to ${RESULT_URL} ..."
dir("${RESULT_DIR}") {
// gather all result files in a folder
powershell """
\$ErrorActionPreference = 'Continue'
Write-Host "Getting the console log..."
Invoke-WebRequest -OutFile console-log.txt -uri "http://jenkins.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText" -ErrorAction "Continue"
Write-Host "Copying build artefacts..."
Copy-Item "${LLVM_DIR}\\choose_projects.log"
Copy-Item "${LLVM_DIR}\\build\\CMakeCache.txt"
Copy-Item "${LLVM_DIR}\\build\\test-results.xml"
"""
}
/// send results to Phabricator
echo "Sending build feedback to Phabricator..."
bat """
python ${SCRIPT_DIR}/phabtalk/phabtalk.py "${PHID}" "${DIFF_ID}" ^
--workspace "${LLVM_DIR}" ^
--conduit-token "${CONDUIT_TOKEN}" ^
--test-result-file "test-results.xml" ^
--host "${PHABRICATOR_HOST}/api/" ^
--results-dir "${RESULT_DIR}" ^
--results-url "${RESULT_URL}" ^
--failures "${failure_message}" ^
--buildresult ${currentBuild.result} ^
--name "windows"
"""
dir("${RESULT_DIR}") {
// upload results to
// Google Cloud Storage
powershell """
Write-Host "Uploading results to GCS..."
\$ErrorActionPreference = 'Continue'
gsutil cp *.* gs://llvm-premerge-checks/results/${MY_BUILD_ID}/
Write-Host "Done."
"""
}
echo "Done."
}
}
}

61
containers/agent-debian-testing-buildkite/Dockerfile
View file

@ -1,61 +0,0 @@
FROM debian:testing
RUN echo "deb [trusted=yes] http://apt.llvm.org/buster/ llvm-toolchain-buster-10 main\n$(cat /etc/apt/sources.list)" > /etc/apt/sources.list ;\
apt-get update ;\
apt-get install -y --no-install-recommends locales \
cmake ninja-build git ca-certificates clang lld ccache python3 build-essential gdb \
clang-tidy clang-format \
python3-psutil zip wget \
openjdk-11-jdk \
python3-pip python3-setuptools \
swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq; \
apt-get clean
# Make python3 default (needed by git-clang-format and others).
RUN rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python
RUN ls /usr/bin
# required for openssh server
RUN mkdir -p /run/sshd
ARG user=buildkite
ARG group=buildkite
ARG uid=1000
ARG gid=1000
ARG AGENT_WORKDIR=/home/${user}/agent
RUN mkdir -p /scripts
COPY start_agent.sh /scripts/
# install python dependencies for the scripts
# ADD will checks that contentent of a file has changed.
ADD "https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/requirements.txt" requirements.txt
RUN pip3 install wheel ; \
pip3 install -r requirements.txt
RUN groupadd -g ${gid} ${group} ;\
useradd -c "buildkite user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} ;\
mkdir /home/${user}/ccache
RUN chown -R ${user}:${user} /home/${user}
WORKDIR /home/${user}
ENV CCACHE_DIR=/mnt/disks/ssd0/ccache
ENV CCACHE_MAXSIZE=20G
# configure locale
RUN sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\
locale-gen
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8
# buildkite
RUN apt-get install -y apt-transport-https gnupg;\
sh -c 'echo deb https://apt.buildkite.com/buildkite-agent stable main > /etc/apt/sources.list.d/buildkite-agent.list' ;\
apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198 ;\
apt-get update ;\
apt-get install -y buildkite-agent
CMD ["/scripts/start_agent.sh"]

31
containers/agent-debian-testing-buildkite/start_agent.sh
View file

@ -1,31 +0,0 @@
#!/bin/bash
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
SSD_ROOT="/mnt/disks/ssd0"
AGENT_ROOT="${SSD_ROOT}/agent"
USER=buildkite-agent
# prepare root folder for Jenkins agent
mkdir -p "${AGENT_ROOT}"
chown -R ${USER}:${USER} "${AGENT_ROOT}"
# prepare folder for ccache
mkdir -p "${CCACHE_DIR}"
chown -R ${USER}:${USER} "${CCACHE_DIR}"
# TODO(kuhnel): wipe the disk(s) on startup
# start the buildkite agent
su buildkite-agent -c "buildkite-agent start --build-path=/mnt/disks/ssd0/agent"

60
containers/agent-debian-testing-ssd/Dockerfile
View file

@ -1,60 +0,0 @@
FROM debian:testing
RUN echo "deb [trusted=yes] http://apt.llvm.org/buster/ llvm-toolchain-buster-10 main\n$(cat /etc/apt/sources.list)" > /etc/apt/sources.list;\
apt-get update ;\
apt-get install -y --no-install-recommends locales \
cmake ninja-build git ca-certificates clang lld ccache python3 build-essential openssh-client\
clang-tidy clang-format gdb \
python3-psutil arcanist zip wget \
openjdk-11-jdk \
python3-pip python3-setuptools \
swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq; \
apt-get clean
# Make python3 default (needed by git-clang-format and others).
RUN rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python
ARG user=jenkins
ARG group=jenkins
ARG uid=1000
ARG gid=1000
ARG AGENT_WORKDIR=/home/${user}/agent
RUN mkdir -p /scripts
# install the swarm client
RUN cd /scripts ;\
wget https://repo.jenkins-ci.org/releases/org/jenkins-ci/plugins/swarm-client/3.17/swarm-client-3.17.jar ;\
mv swarm-client-3.17.jar swarm-client.jar
COPY start_agent.sh report_results.sh /scripts/
# store SSH known hosts for github, required for ssh authentication
RUN mkdir -p /home/${user}/.ssh
COPY known_hosts /home/${user}/.ssh/known_hosts
# install python dependencies for the scripts
# ADD will checks that contentent of a file has changed.
ADD "https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/requirements.txt" requirements.txt
RUN pip3 install wheel ; \
pip3 install -r requirements.txt
RUN groupadd -g ${gid} ${group} ;\
useradd -c "Jenkins user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} ;\
mkdir /home/${user}/ccache
RUN chown -R ${user}:${user} /home/${user}
WORKDIR /home/${user}
ENV CCACHE_DIR=/mnt/disks/ssd0/ccache
ENV CCACHE_MAXSIZE=20G
# configure locale
RUN sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\
locale-gen
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8
CMD ["/scripts/start_agent.sh"]

4
containers/agent-debian-testing-ssd/known_hosts
View file

@ -1,4 +0,0 @@
|1|bJzGXTLCQ4FZRpq+RQu+NfQOugI=|45lSmEMlpfJx7897p2Th4tZj6rM= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==|1|CA5hL0xfZtRH24/h4PieLzQaV5E=|gEuFUpdJK9mwpp1PH8RFi3DFLis= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
|1|K6qo2Wrdv5gQipncPel2cFaNT/w=|k+coolWLGXsJ/oM4G9PBY3GLJQQ= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
|1|hIGbHg7+Z8TQrZ/OEiRxa7f9TZs=|h6iCbIE5wV5wjMo4auBXVXgNWtU= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
|1|PdWOrYv48xcuktJiKm97UQTg2d0=|zZImMkWTMV8HfZAUv34OvQvKyds= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==

38
containers/agent-debian-testing-ssd/report_results.sh
View file

@ -1,38 +0,0 @@
#!/bin/bash
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eux
SUCCESS="$1"
DIFF_ID="$2"
BUILD_ID="$3"
URL="https://reviews.llvm.org"
if [ "${SUCCESS}" == "0" ]
then
MSG="SUCCESSFUL"
else
MSG="FAILED"
fi
arc --conduit-uri=${URL} call-conduit differential.revision.edit <<EOF
{
"transactions": [{
"type": "comment",
"value": "check-all ${MSG}! \n[cmake.log](http://results.llvm-merge-guard.org/${BUILD_ID}/cmake.log)\n[ninja_check_all.log](http://results.llvm-merge-guard.org/${BUILD_ID}/ninja_check_all.log)"
}],
"objectIdentifier":"${DIFF_ID}"
}
EOF

41
containers/agent-debian-testing-ssd/start_agent.sh
View file

@ -1,41 +0,0 @@
#!/bin/bash
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
SSD_ROOT="/mnt/disks/ssd0"
AGENT_ROOT="${SSD_ROOT}/agent"
SSH_KEY_SOURCE="/github-ssh-key"
SSH_KEY_TARGET="/home/jenkins/.ssh"
# wipe the local cache on restart
rm -rf "$SSD_ROOT"
# prepare root folder for Jenkins agent
mkdir -p "${AGENT_ROOT}"
chown -R jenkins:jenkins "${AGENT_ROOT}"
# prepare folder for ccache
mkdir -p "${CCACHE_DIR}"
chown -R jenkins:jenkins "${CCACHE_DIR}"
# copy ssh keys to user jenkins
mkdir -p ${SSH_KEY_TARGET}
cp ${SSH_KEY_SOURCE}/* ${SSH_KEY_TARGET}
chmod 700 ${SSH_KEY_TARGET}
chmod 600 ${SSH_KEY_TARGET}/*
chown -R jenkins:jenkins ${SSH_KEY_TARGET}
# start swarm agent as user jenkins
# description of arguments: https://wiki.jenkins.io/display/JENKINS/Swarm+Plugin
su jenkins -c "java -jar /scripts/swarm-client.jar -master http://jenkins-ui.jenkins.svc.cluster.local:8080 -executors 1 -fsroot ${AGENT_ROOT} -labels linux"

26
containers/agent-windows-jenkins/Dockerfile
View file

@ -1,26 +0,0 @@
# escape=`
# use windows server core image
ARG agent_windows_version
FROM gcr.io/llvm-premerge-checks/agent-windows-vs2019:${agent_windows_version}
# install java
RUN choco install -y openjdk
# get Jenkins swarm plugin
ENV SWARM_PLUGIN_URL="https://repo.jenkins-ci.org/releases/org/jenkins-ci/plugins/swarm-client/3.17/swarm-client-3.17.jar"
ENV SWARM_PLUGIN_JAR="C:\jenkins\swarm-client.jar"
RUN powershell -NoProfile -InputFormat None -Command `
mkdir c:\jenkins ; `
Invoke-WebRequest -Uri %SWARM_PLUGIN_URL% -OutFile %SWARM_PLUGIN_JAR%
# install gsutils to access Google Cloud Storage
RUN pip install gsutil
VOLUME C:\credentials
# temporary directory, can be mounted on host if required
VOLUME C:\Temp
# start swarm plugin
COPY start_agent.ps1 c:\jenkins
CMD ["C:\\jenkins\\start_agent.ps1"]

BIN
containers/agent-windows-jenkins/VERSION
View file

Binary file not shown.

48
containers/agent-windows-jenkins/start_agent.ps1
View file

@ -1,48 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
$JENKINS_SERVER="jenkins.local"
$AGENT_ROOT="C:\ws"
# TODO(kuhnel): The autentication does not work!
# trying to copy the .boto file instead
# authenticate gsutil
# Write-Output "C:\credentials\build-agent-results_key.json`nllvm-premerge-checks`n`n" | gsutil config -e
Copy-Item "C:\credentials\.boto" "C:\Users\ContainerAdministrator\.boto"
$env:TEMP="${$AGENT_ROOT}\TEMP"
$env:TMP="${env:TEMP}"
# set local cache folder for sccache
$env:SCCACHE_DIR="C:\ws\sccache"
# Start sccache server and keep it running to avoid problems with Jenkins
# https://github.com/mozilla/sccache/blob/master/docs/Jenkins.md
$env:SCCACHE_IDLE_TIMEOUT="0"
# wipe cache at startup, otherwise it will time out
Remove-Item -Recurse -Force $env:SCCACHE_DIR
sccache --start-server
if ($lastexitcode -ne 0) {
Write-Error "Failed to start sccache server."
exit $lastexitcode
}
# start Jenkins agent
java -jar ${env:SWARM_PLUGIN_JAR} `
-master http://${JENKINS_SERVER}:8080 `
-executors 1 `
-fsroot ${AGENT_ROOT} `
-labels "windows vs2019 cores_${env:NUMBER_OF_PROCESSORS}" `
-name ${env:PARENT_HOSTNAME}

48
containers/agent-windows-vs2017/Dockerfile
View file

@ -1,48 +0,0 @@
# escape=`
# use windows server core image
ARG windows_version=ltsc2019
FROM mcr.microsoft.com/windows/servercore:${windows_version}
# install chocolately as package manager
RUN powershell -NoProfile -InputFormat None -Command `
iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')) ; `
choco feature disable --name showDownloadProgress
# install Visual Studio build tools
RUN powershell -NoProfile -InputFormat None -Command `
choco install visualcpp-build-tools `
--version 15.0.26228.20170424 -y --params "'/IncludeOptional'" ;`
Write-Host 'Waiting for Visual C++ Build Tools to finish'; `
Wait-Process -Name vs_installer
# install other tools as described in https://llvm.org/docs/GettingStartedVS.html
# and a few more that were not documented...
RUN choco install -y git
RUN choco install -y cmake --version 3.15.4
RUN choco install -y python3
RUN choco install -y ninja
RUN choco install -y gnuwin
# install perl, required for OpenMP
# RUN choco install -y strawberryperl
RUN pip install psutil
# configure Python encoding
ENV PYTHONIOENCODING=UTF-8
# update the path variable
RUN powershell -NoProfile -InputFormat None -Command `
$path = $env:path + ';c:\Program Files (x86)\GnuWin32\bin;C:\Program Files\CMake\bin'; `
Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment\' -Name Path -Value $path
# use this folder to store the worksapce'
VOLUME C:\ws
WORKDIR C:\ws
# support long file names during git checkout
RUN git config --system core.longpaths true & `
git config --global core.autocrlf false
# Start developer command prompt with any other commands specified.
#ENTRYPOINT "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat" &
CMD cmd

BIN
containers/agent-windows-vs2017/VERSION
View file

Binary file not shown.

2
containers/build_run.sh
View file

@ -14,7 +14,7 @@
# limitations under the License. # limitations under the License.
# Starts a new instances of a docker image. Example: # Starts a new instances of a docker image. Example:
# sudo build_run.sh agent-debian-testing-ssd /bin/bash # sudo build_run.sh buildkite-premerge-debian /bin/bash
set -eux set -eux
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

4
containers/jenkins-master/Dockerfile
View file

@ -1,4 +0,0 @@
FROM jenkins/jenkins:lts
# disable CSRF protection, otherwise Phabricator can't trigger builds
ENV JENKINS_OPTS "-D hudson.security.csrf.GlobalCrumbIssuerConfiguration.DISABLE_CSRF_PROTECTION=true"

16
containers/nginx-results/Dockerfile
View file

@ -1,16 +0,0 @@
FROM nginx:1.17
RUN set -e ;\
apt-get update ;\
apt-get install -y --no-install-recommends python3 cron;\
apt-get clean
RUN mkdir -p /scripts
COPY run_nginx.sh clean_results.py /scripts/
COPY default.conf /etc/nginx/conf.d/
COPY crontab /etc/cron.d/clean_results
RUN chmod 0644 /etc/cron.d/clean_results
RUN service cron start
CMD ["/scripts/run_nginx.sh"]

40
containers/nginx-results/clean_results.py
View file

@ -1,40 +0,0 @@
#!/usr/bin/python3
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
import os
import shutil
ROOT_DIR = '/mnt/nfs/results'
MAX_AGE = datetime.timedelta(days=90)
MAX_AGE_BIN = datetime.timedelta(days=3)
now = datetime.datetime.now()
for folder in [f for f in os.listdir(ROOT_DIR)]:
fullpath = os.path.join(ROOT_DIR, folder)
if not os.path.isdir(fullpath):
continue
print(fullpath)
binpath = os.path.join(ROOT_DIR, folder, 'binaries')
stats=os.stat('/tmp')
created = datetime.datetime.fromtimestamp(stats.st_mtime)
print(created)
if created + MAX_AGE < now:
print("Deleting all results: {}".format(fullpath))
shutil.rmtree(fullpath)
elif os.path.exists(binpath) and created + MAX_AGE_BIN < now:
print("Deleting binaries: {}".format(binpath))
shutil.rmtree(binpath)

2
containers/nginx-results/crontab
View file

@ -1,2 +0,0 @@
* * * * * /scripty/clean_results.py
# Mandatory blank line

38
containers/nginx-results/default.conf
View file

@ -1,38 +0,0 @@
server {
listen 80;
server_name results.llvm-merge-guard.org;
#charset koi8-r;
#access_log /var/log/nginx/host.access.log main;
location / {
root /mnt/nfs/results;
index index.html index.htm;
autoindex on;
}
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
}
# proxy the PHP scripts to Apache listening on 127.0.0.1:80
#
#location ~ \.php$ {
# proxy_pass http://127.0.0.1;
#}
# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
#
#location ~ \.php$ {
# root html;
# fastcgi_pass 127.0.0.1:9000;
# fastcgi_index index.php;
# fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name;
# include fastcgi_params;
#}
# deny access to .htaccess files, if Apache's document root
# concurs with nginx's one
#
#location ~ /\.ht {
# deny all;
#}
}

3
containers/nginx-results/index.html
View file

@ -1,3 +0,0 @@
<HTML>
Hello results!
</HTML>

20
containers/nginx-results/run_nginx.sh
View file

@ -1,20 +0,0 @@
#!/bin/bash
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eux
mkdir -p /mnt/nfs/results
chmod 777 /mnt/nfs/results
nginx -g "daemon off;"

0
docs/buildkite.md → docs/decisions/1_migrate_from_jenkins_to_buildkite.md
View file

5
docs/development.md
View file

@ -99,6 +99,8 @@ proxy](https://kubernetes.github.io/ingress-nginx/deploy/#gce-gke) and
[certificate [certificate
manager](http://docs.cert-manager.io/en/latest/getting-started/install/kubernetes.html). manager](http://docs.cert-manager.io/en/latest/getting-started/install/kubernetes.html).
[lets-encrypt configuration](../kubernetes/cert-issuer.yaml).
Access to the service is restricted with basic HTTP auth. It's configured with Access to the service is restricted with basic HTTP auth. It's configured with
k8s secret 'http-auth' in 'buildkite' namespace (see [how to update k8s secret 'http-auth' in 'buildkite' namespace (see [how to update
auth](playbooks.md#update-http-auth-credentials)) auth](playbooks.md#update-http-auth-credentials))
@ -177,4 +179,7 @@ chmod +x buildkite-agent-metrics-linux-amd64
nohup ./buildkite-agent-metrics-linux-amd64 -token XXXX -interval 30s -backend statsd & nohup ./buildkite-agent-metrics-linux-amd64 -token XXXX -interval 30s -backend statsd &
``` ```
Metrics are exported as "custom/statsd/gauge".
TODO: update "Testing scripts locally" playbook on how to run Linux build locally with Docker. TODO: update "Testing scripts locally" playbook on how to run Linux build locally with Docker.
TODO: migrate 'builkite-monitoring' to k8s deployment.

83
docs/playbooks.md
View file

@ -1,24 +1,22 @@
- [Playbooks](#playbooks) - [Playbooks](#playbooks)
* [deployment to a clean infrastructure](#deployment-to-a-clean-infrastructure) * [deployment to a clean infrastructure](#deployment-to-a-clean-infrastructure)
* [creating basic authentication for reverse proxy](#creating-basic-authentication-for-reverse-proxy)
* [Creating docker containers on Windows](#creating-docker-containers-on-windows) * [Creating docker containers on Windows](#creating-docker-containers-on-windows)
* [Spawning a new windows agent](#spawning-a-new-windows-agent) * [Spawning a new windows agent](#spawning-a-new-windows-agent)
+ [Buildkite](#buildkite) + [Buildkite](#buildkite)
+ [Jenkins](#jenkins)
* [Testing scripts locally](#testing-scripts-locally) * [Testing scripts locally](#testing-scripts-locally)
* [Custom environment variables](#custom-environment-variables) * [Custom environment variables](#custom-environment-variables)
* [Testing changes before merging](#testing-changes-before-merging) * [Testing changes before merging](#testing-changes-before-merging)
- [Update HTTP auth credentials](#update-http-auth-credentials) * [Update HTTP auth credentials](#update-http-auth-credentials)
- [Phabricator integration](#phabricator-integration)
* [Herald](#herald)
* [Harbormaster](#harbormaster)
# Playbooks # Playbooks
## deployment to a clean infrastructure ## deployment to a clean infrastructure
General remarks: General remarks:
* GCP does not route any traffic to your services unless the service is "healthy". It might take a few minutes after startup before the services is classified as healthy. Until then you will only see some generic error message. * GCP does not route any traffic to your services unless the service is
"healthy". It might take a few minutes after startup before the services is
classified as healthy. Until then, you will only see some generic error
message.
These are the steps to set up the build server on a clean infrastructure: These are the steps to set up the build server on a clean infrastructure:
1. Configure the tools on your local machine: 1. Configure the tools on your local machine:
@ -38,18 +36,6 @@ These are the steps to set up the build server on a clean infrastructure:
cd kubernetes/cluster cd kubernetes/cluster
./cluster_create.sh ./cluster_create.sh
``` ```
1. Create the disk storage, if it does not yet exist:
```bash
cd kubernetes/cluster
./disk_create.sh
```
1. SSH into the VM instance mounting the volume, find the mount point and then set
```bash
# go to the mount point of the volume
cd /var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/jenkins-home
# change the permissions
sudo chmod a+rwx
```
1. Push the docker images to gcr.io: 1. Push the docker images to gcr.io:
```bash ```bash
cd containers cd containers
@ -63,17 +49,6 @@ These are the steps to set up the build server on a clean infrastructure:
``` ```
1. Configure it 1. Configure it
## creating basic authentication for reverse proxy
1. create auth file, based on [ingress-nginx documentation](https://github.com/kubernetes/ingress-nginx/tree/master/docs/examples/auth/basic)
```bash
cd kubernetes/reverse-proxy
htpasswd -c auth <username>
# enter password at prompt
# add more users as required
kubectl create secret generic proxy-auth --from-file=auth --namespace=jenkins
```
## Creating docker containers on Windows ## Creating docker containers on Windows
If you want to build/update/test docker container for Windows, you need to do this on a Windows machine. If you want to build/update/test docker container for Windows, you need to do this on a Windows machine.
@ -116,8 +91,8 @@ Here are the instructions to set up such a machine on GCP.
cd c:\ cd c:\
git clone https://github.com/google/llvm-premerge-checks git clone https://github.com/google/llvm-premerge-checks
cd llvm-premerge-checks\containers cd llvm-premerge-checks\containers
.\build_deploy.ps1 agent-windows-buildkite # or agent-windows-jenkins .\build_deploy.ps1 agent-windows-buildkite
c:\llvm-premerge-check\scripts\windows_agent_start_buildkite.ps1 # or windows_agent_start_jenkins.ps1 c:\llvm-premerge-check\scripts\windows_agent_start_buildkite.ps1
``` ```
## Spawning a new windows agent ## Spawning a new windows agent
@ -156,21 +131,9 @@ git clone https://github.com/google/llvm-premerge-checks.git C:\llvm-premerge-ch
schtasks.exe /create /tn "Start Buildkite agent" /ru SYSTEM /SC ONSTART /DELAY 0005:00 /tr "powershell -command 'C:\llvm-premerge-checks\scripts\windows_agent_start_buildkite.ps1 -workdir c:\ws'" schtasks.exe /create /tn "Start Buildkite agent" /ru SYSTEM /SC ONSTART /DELAY 0005:00 /tr "powershell -command 'C:\llvm-premerge-checks\scripts\windows_agent_start_buildkite.ps1 -workdir c:\ws'"
``` ```
### Jenkins
1. Create `c:\credentials` folder with `build-agent-results_key.json` to access cloud storage copy from one of the existing machines.
1. Run
```powershell
git clone https://github.com/google/llvm-premerge-checks.git "c:\llvm-premerge-checks"
C:\llvm-premerge-checks\scripts\windows_agent_start_buildkite.ps1 [-testing] [-version latest]
```
Metrics are exported as "custom/statsd/gauge".
## Testing scripts locally ## Testing scripts locally
Build and run agent docker image `sudo ./containers/build_run.sh agent-debian-testing-ssd /bin/bash`. Build and run agent docker image `sudo ./containers/build_run.sh buildkite-premerge-debian /bin/bash`.
Within a container set environment variables similar to [pipeline](https://github.com/google/llvm-premerge-checks/blob/master/Jenkins/Phabricator-pipeline/Jenkinsfile).
Additionally set `WORKSPACE`, `PHID` and `DIFF_ID` parameters. Set `CONDUIT_TOKEN` with your personal one from `https://reviews.llvm.org/settings/user/<USERNAME>/page/apitokens/`. Additionally set `WORKSPACE`, `PHID` and `DIFF_ID` parameters. Set `CONDUIT_TOKEN` with your personal one from `https://reviews.llvm.org/settings/user/<USERNAME>/page/apitokens/`.
@ -200,7 +163,7 @@ It's recommended to test even smallest changes before committing them to the `ma
`scripts_branch="my-feature"` (see also "Custom environment variables" for other options above). To test "premerge-tests" pipeline pick an existing build and copy parameters from it, omitting "ph_target_phid", namely: "ph_build_id", "ph_buildable_diff", "ph_buildable_revision", "ph_initiator_phid" and "scripts_branch" variables. `scripts_branch="my-feature"` (see also "Custom environment variables" for other options above). To test "premerge-tests" pipeline pick an existing build and copy parameters from it, omitting "ph_target_phid", namely: "ph_build_id", "ph_buildable_diff", "ph_buildable_revision", "ph_initiator_phid" and "scripts_branch" variables.
1. Wait for build to complete and maybe attach a link to it to your PR. 1. Wait for build to complete and maybe attach a link to it to your PR.
# Update HTTP auth credentials ## Update HTTP auth credentials
To update e.g. buildkite http-auth: To update e.g. buildkite http-auth:
```shell script ```shell script
@ -213,31 +176,3 @@ htpasswd -b auth <user> <pass>
kubectl delete secret http-auth -n buildkite kubectl delete secret http-auth -n buildkite
kubectl create secret generic http-auth -n buildkite --from-file=./auth kubectl create secret generic http-auth -n buildkite --from-file=./auth
``` ```
# Phabricator integration
The general flow for builds on Phabricator is:
1. A user uploads a *Diff* (=patch) to a *Revision* (set of Diffs with comments and buildstatus, ... ).
2. A *Herald* checks if one of the *rules* matches this event.
3. You can use the rules to trigger a *Build* in *Harbormaster*.
4. Harbor sends an HTTP request to the Jenkins server.
5. Jenkins executes the build. In the last step of the build, a script is uploading the results to Phabricator.
6. Phabricator sets the build status and displays the results.
## Herald
We currently have these Herald rules to configure the builds:
* Triggering builds for everyone:
* [H576](https://reviews.llvm.org/H576) This will only trigger for non-beta testers.
* Triggering the beta-test builds:
* [H511](https://reviews.llvm.org/H511) or the beta testers, this is for testing new features.
* [H552](https://reviews.llvm.org/H552) for all changes to MLIR (archived)
* [H527](https://reviews.llvm.org/H527) for all changes to clang-extra-tools (archived)
You can *archive* a rule to disable it.
## Harbormaster
We have these build plans in Harbormaster:
* [Plan 4](https://reviews.llvm.org/harbormaster/plan/4/) Builds for everyone
* [Plan 3](https://reviews.llvm.org/harbormaster/plan/3/) Builds for beta testers

20
kubernetes/buildkite/kustomization.yaml
View file

@ -1,20 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: jenkins
resources:
- deployment.yaml

10
kubernetes/buildkite/deployment.yaml → kubernetes/buildkite/linux-agents.yaml
View file

@ -15,7 +15,7 @@
apiVersion: apps/v1 apiVersion: apps/v1
kind: Deployment kind: Deployment
metadata: metadata:
name: premerge-debian name: linux-agents
namespace: buildkite namespace: buildkite
spec: spec:
replicas: 4 replicas: 4
@ -53,8 +53,12 @@ spec:
secretKeyRef: secretKeyRef:
name: buildkite-agent-token name: buildkite-agent-token
key: token key: token
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: BUILDKITE_AGENT_TAGS - name: BUILDKITE_AGENT_TAGS
value: "queue=linux" value: "queue=linux,name=$(POD_NAME)"
- name: BUILDKITE_BUILD_PATH - name: BUILDKITE_BUILD_PATH
value: "/mnt/disks/ssd0/agent" value: "/mnt/disks/ssd0/agent"
- name: CONDUIT_TOKEN - name: CONDUIT_TOKEN
@ -72,4 +76,4 @@ spec:
secret: secret:
secretName: github-ssh secretName: github-ssh
nodeSelector: nodeSelector:
cloud.google.com/gke-nodepool: jenkins-agents cloud.google.com/gke-nodepool: linux-agents

0
kubernetes/reverse-proxy/Issuer.yaml → kubernetes/cert-issuer.yaml
View file

5
kubernetes/cluster_create.sh
View file

@ -25,10 +25,9 @@ source "${ROOT_DIR}/k8s_config"
gcloud container clusters create $GCP_CLUSTER --zone $GCP_ZONE \ gcloud container clusters create $GCP_CLUSTER --zone $GCP_ZONE \
--machine-type=n1-standard-4 --num-nodes=1 --machine-type=n1-standard-4 --num-nodes=1
# Jenkins agents with local ssd # Linux agents node pool with local ssd.
# as per instructions
# https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/local-ssd # https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/local-ssd
gcloud container node-pools create jenkins-agents --cluster $GCP_CLUSTER --zone $GCP_ZONE \ gcloud container node-pools create linux-agents --cluster $GCP_CLUSTER --zone $GCP_ZONE \
--machine-type=n1-standard-32 --num-nodes=2 --local-ssd-count=1 --machine-type=n1-standard-32 --num-nodes=2 --local-ssd-count=1
# created separate cluster for windows, as we need "ip-alias" enabled # created separate cluster for windows, as we need "ip-alias" enabled

21
kubernetes/deploy.sh
View file

@ -1,21 +0,0 @@
#!/bin/bash
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eux
# nfs needs to be deployed first, as it creates the presistent volumes
kubectl apply -f nfs.yaml
kubectl apply -k results-nginx
kubectl apply -f jenkins.yaml

40
kubernetes/disk_create.sh
View file

@ -1,40 +0,0 @@
#!/bin/bash
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eux
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
ROOT_DIR="$(dirname ${DIR})"
# get config options
source "${ROOT_DIR}/k8s_config"
gcloud compute disks create jenkins-home \
--description="storage for jenkins master" \
--size=200GB \
--type=pd-standard \
--zone=${GCP_ZONE} \
gcloud compute disks create jenkins-home-staging \
--description="storage for jenkins master (staging)" \
--size=20GB \
--type=pd-standard \
--zone=${GCP_ZONE} \
gcloud compute disks create results \
--description="storage build results" \
--size=20GB \
--type=pd-standard \
--zone=${GCP_ZONE}

20
kubernetes/disk_delete.sh
View file

@ -1,20 +0,0 @@
#!/bin/bash
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eux
# to detatch:
# gcloud compute instances detach-disk --disk=jenkins-home <HOST>
gcloud compute disks delete jenkins-home

62
kubernetes/jenkins-staging/Deployment.yaml
View file

@ -1,62 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: jenkins-staging
namespace: jenkins
spec:
replicas: 1
template:
metadata:
labels:
app: master-staging
spec:
hostname: jenkins-master-staging
containers:
- name: master-staging
image: gcr.io/llvm-windows-development/jenkins-master
ports:
- containerPort: 8080
- containerPort: 50000
readinessProbe:
httpGet:
path: /login
port: 8080
periodSeconds: 10
timeoutSeconds: 5
successThreshold: 2
failureThreshold: 5
env:
- name: JAVA_OPTS
value: '-Xmx1400m'
volumeMounts:
- mountPath: /var/jenkins_home
name: jenkins-home-staging
resources:
limits:
cpu: 500m
memory: 1500Mi
requests:
cpu: 500m
memory: 1500Mi
volumes:
- name: jenkins-home-staging
gcePersistentDisk:
pdName: jenkins-home-staging
fsType: ext4
nodeSelector:
cloud.google.com/gke-nodepool: default-pool
---

29
kubernetes/jenkins-staging/PersistentVolume.yaml
View file

@ -1,29 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
kind: PersistentVolume
metadata:
name: jenkins-home-staging
labels:
failure-domain.beta.kubernetes.io/zone: us-central1-a
spec:
capacity:
storage: 20Gi
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
gcePersistentDisk:
pdName: jenkins-home-staging
fsType: ext4

30
kubernetes/jenkins-staging/Services.yaml
View file

@ -1,30 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
kind: Service
apiVersion: v1
metadata:
name: jenkins-ui-staging
namespace: jenkins
spec:
type: NodePort
selector:
app: master-staging
ports:
- protocol: TCP
port: 8080
targetPort: 8080
name: ui
---

22
kubernetes/jenkins-staging/kustomization.yaml
View file

@ -1,22 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: jenkins
resources:
- PersistentVolume.yaml
- Deployment.yaml
- Services.yaml

195
kubernetes/jenkins.yaml
View file

@ -1,195 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
apiVersion: v1
kind: Namespace
metadata:
name: jenkins
labels:
name: jenkins
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: default
namespace: jenkins
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: jenkins-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: default
namespace: jenkins
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: jenkins-home
labels:
failure-domain.beta.kubernetes.io/zone: us-central1-a
spec:
capacity:
storage: 20Gi
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
gcePersistentDisk:
pdName: jenkins-home
fsType: ext4
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: jenkins
namespace: jenkins
spec:
replicas: 1
template:
metadata:
labels:
app: master
spec:
hostname: jenkins-master
containers:
- name: master
image: jenkins/jenkins:2.190.3
ports:
- containerPort: 8080
- containerPort: 50000
readinessProbe:
httpGet:
path: /login
port: 8080
periodSeconds: 10
timeoutSeconds: 5
successThreshold: 2
failureThreshold: 5
env:
- name: JAVA_OPTS
value: '-Xmx1400m'
volumeMounts:
- mountPath: /var/jenkins_home
name: jenkins-home
resources:
limits:
cpu: "4"
memory: 4000Mi
requests:
cpu: 500m
memory: 2000Mi
volumes:
- name: jenkins-home
gcePersistentDisk:
pdName: jenkins-home
fsType: ext4
nodeSelector:
cloud.google.com/gke-nodepool: default-pool
---
kind: Service
apiVersion: v1
metadata:
name: jenkins-ui
namespace: jenkins
spec:
type: NodePort
selector:
app: master
ports:
- protocol: TCP
port: 8080
targetPort: 8080
name: ui
- protocol: TCP
port: 50000
targetPort: 50000
name: agent
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: nfs-jenkins
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteMany
nfs:
server: nfs-service.results.svc.cluster.local
path: "/exports"
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: nfs-jenkins
namespace: jenkins
spec:
accessModes:
- ReadWriteMany
storageClassName: ""
resources:
requests:
storage: 10Gi
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: agent-debian-testing-ssd
namespace: jenkins
spec:
replicas: 2
template:
metadata:
labels:
app: agent-debian-testing-ssd
spec:
containers:
- name: agent-debian-testing-clang8
image: gcr.io/llvm-premerge-checks/agent-debian-testing-ssd
ports:
- containerPort: 22
resources:
limits:
cpu: 30
memory: 45Gi
requests:
cpu: 30
memory: 45Gi
volumeMounts:
- name: nfs-pvc
mountPath: /mnt/nfs
- name: ssd
mountPath: /mnt/disks/ssd0
- mountPath: /github-ssh-key
name: github-ssh-key
volumes:
- name: nfs-pvc
persistentVolumeClaim:
claimName: nfs-jenkins
- name: ssd
hostPath:
# directory location on host
path: /mnt/disks/ssd0
type: Directory
- name: github-ssh-key
secret:
secretName: github-ssh-key
nodeSelector:
cloud.google.com/gke-nodepool: jenkins-agents

108
kubernetes/nfs.yaml
View file

@ -1,108 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
apiVersion: v1
kind: Namespace
metadata:
name: results
labels:
name: results
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: nfs-server
namespace: results
spec:
replicas: 1
selector:
matchLabels:
role: nfs-server
template:
metadata:
labels:
role: nfs-server
spec:
containers:
- name: nfs-server
image: gcr.io/google_containers/volume-nfs
ports:
- name: nfs
containerPort: 2049
- name: mountd
containerPort: 20048
- name: rpcbind
containerPort: 111
securityContext:
privileged: true
volumeMounts:
- mountPath: /exports
name: result-pvc
resources:
limits:
cpu: 500m
memory: 500Mi
requests:
cpu: 500m
memory: 500Mi
volumes:
- name: result-pvc
gcePersistentDisk:
pdName: results-new
fsType: ext4
nodeSelector:
cloud.google.com/gke-nodepool: default-pool
---
apiVersion: v1
kind: Service
metadata:
name: nfs-service
namespace: results
spec:
ports:
- name: nfs
port: 2049
- name: mountd
port: 20048
- name: rpcbind
port: 111
selector:
role: nfs-server
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: nfs
spec:
capacity:
storage: 20Gi
accessModes:
- ReadWriteMany
nfs:
server: nfs-service.results.svc.cluster.local
path: "/exports"
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: nfs
namespace: results
spec:
accessModes:
- ReadWriteMany
storageClassName: ""
resources:
requests:
storage: 1Gi

2
kubernetes/phabricator-proxy/kustomization.yaml
View file

@ -14,7 +14,7 @@
apiVersion: kustomize.config.k8s.io/v1beta1 apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization kind: Kustomization
namespace: jenkins namespace: buildkite
resources: resources:
- Deployment.yaml - Deployment.yaml
- Services.yaml - Services.yaml

58
kubernetes/results-nginx/Deployment.yaml
View file

@ -1,58 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-results
namespace: jenkins
spec:
selector:
matchLabels:
app: nginx-results
replicas: 1
template:
metadata:
labels:
app: nginx-results
spec:
containers:
- name: nginx-results
image: gcr.io/llvm-premerge-checks/nginx-results
ports:
- containerPort: 80
readinessProbe:
httpGet:
path: /
port: 80
periodSeconds: 10
timeoutSeconds: 5
successThreshold: 2
failureThreshold: 5
volumeMounts:
- mountPath: /mnt/nfs
name: nfs-pvc
resources:
limits:
cpu: 500m
memory: 1500Mi
requests:
cpu: 500m
memory: 1500Mi
volumes:
- name: nfs-pvc
persistentVolumeClaim:
claimName: nfs-jenkins
nodeSelector:
cloud.google.com/gke-nodepool: default-pool

28
kubernetes/results-nginx/Services.yaml
View file

@ -1,28 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
kind: Service
apiVersion: v1
metadata:
name: nginx-results
namespace: jenkins
spec:
# type: LoadBalancer
# externalTrafficPolicy: Cluster
selector:
app: nginx-results
ports:
- protocol: TCP
port: 80
targetPort: 80

21
kubernetes/results-nginx/kustomization.yaml
View file

@ -1,21 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: jenkins
resources:
- Deployment.yaml
- Services.yaml

1
kubernetes/reverse-proxy/.gitignore vendored
View file

@ -1 +0,0 @@
auth

120
kubernetes/reverse-proxy/Ingress.yaml
View file

@ -1,120 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: nginx-ingress-prod-jenkins
namespace: jenkins
annotations:
# static IP assignment not working. Not sure why.
kubernetes.io/ingress.global-static-ip-name: "web-static-ip"
kubernetes.io/ingress.class: "nginx"
cert-manager.io/cluster-issuer: "letsencrypt-prod"
nginx.ingress.kubernetes.io/auth-type: basic
nginx.ingress.kubernetes.io/auth-secret: proxy-auth
nginx.ingress.kubernetes.io/auth-realm: "Authentication Required - LLVM pre-merge checks"
nginx.ingress.kubernetes.io/configuration-snippet: |
proxy_set_header Authorization $remote_user;
spec:
tls:
- secretName: jenkins-prod-tls
hosts:
- jenkins.llvm-merge-guard.org
rules:
- host: jenkins.llvm-merge-guard.org
http:
paths:
- backend:
serviceName: jenkins-ui
servicePort: 8080
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: nginx-ingress-prod-results
namespace: jenkins
annotations:
# static IP assignment not working. Not sure why.
kubernetes.io/ingress.global-static-ip-name: "web-static-ip"
kubernetes.io/ingress.class: "nginx"
cert-manager.io/cluster-issuer: "letsencrypt-prod"
spec:
tls:
- secretName: results-prod-tls
hosts:
- results.llvm-merge-guard.org
rules:
- host: results.llvm-merge-guard.org
http:
paths:
- backend:
serviceName: nginx-results
servicePort: 80
# ---
# apiVersion: extensions/v1beta1
# kind: Ingress
# metadata:
# name: nginx-ingress-staging-jenkins
# namespace: jenkins
# annotations:
# # static IP assignment not working. Not sure why.
# kubernetes.io/ingress.global-static-ip-name: "web-static-ip"
# kubernetes.io/ingress.class: "nginx"
# cert-manager.io/cluster-issuer: "letsencrypt-staging"
# nginx.ingress.kubernetes.io/auth-type: basic
# nginx.ingress.kubernetes.io/auth-secret: proxy-auth
# nginx.ingress.kubernetes.io/auth-realm: "Authentication Required - LLVM pre-merge checks"
# nginx.ingress.kubernetes.io/configuration-snippet: |
# proxy_set_header Authorization $remote_user;
# spec:
# tls:
# - secretName: jenkins-staging-tls
# hosts:
# - jenkins.staging.llvm-merge-guard.org
# rules:
# - host: jenkins.staging.llvm-merge-guard.org
# http:
# paths:
# - backend:
# serviceName: jenkins-ui-staging
# servicePort: 8080
# ---
# apiVersion: extensions/v1beta1
# kind: Ingress
# metadata:
# name: nginx-ingress-staging-results
# namespace: jenkins
# annotations:
# # static IP assignment not working. Not sure why.
# kubernetes.io/ingress.global-static-ip-name: "web-static-ip"
# kubernetes.io/ingress.class: "nginx"
# cert-manager.io/cluster-issuer: "letsencrypt-staging"
# spec:
# tls:
# - secretName: results-staging-tls
# hosts:
# - results.staging.llvm-merge-guard.org
# rules:
# - host: results.staging.llvm-merge-guard.org
# http:
# paths:
# - backend:
# serviceName: nginx-results
# servicePort: 80
# ---

6
kubernetes/reverse-proxy/Namespace.yaml
View file

@ -1,6 +0,0 @@
apiVersion: v1
kind: Namespace
metadata:
name: jenkins
labels:
name: jenkins

25
kubernetes/reverse-proxy/add_user.sh
View file

@ -1,25 +0,0 @@
#!/bin/bash
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eux
USER_NAME="${1}"
AUTH_FILE="${HOME}/.llvm-premerge-checks/auth"
if [ -f "${AUTH_FILE}" ] ; then
htpasswd "${AUTH_FILE}" "${USER_NAME}"
else
mkdir -p "$(dirname "${AUTH_FILE}")"
htpasswd -c "${AUTH_FILE}" "${USER_NAME}"
fi

30
kubernetes/reverse-proxy/basic.sh
View file

@ -1,30 +0,0 @@
#!/bin/bash
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eux
# basic setup of the reverse proxy based on
# https://kubernetes.github.io/ingress-nginx/deploy/
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/mandatory.yaml
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/cloud-generic.yaml
# install certmanager based on
# http://docs.cert-manager.io/en/latest/getting-started/install/kubernetes.html
kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.15.0/cert-manager.yaml
kubectl create clusterrolebinding cluster-admin-binding \
--clusterrole=cluster-admin \
--user=$(gcloud config get-value core/account)

23
kubernetes/reverse-proxy/kustomization.yaml
View file

@ -1,23 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: jenkins
resources:
- Namespace.yaml
- Issuer.yaml
- Certificates.yaml
- Ingress.yaml

29
kubernetes/reverse-proxy/update_auth.sh
View file

@ -1,29 +0,0 @@
#!/bin/bash
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eux
AUTH_FILE="${HOME}/.llvm-premerge-checks/auth"
if [ ! -f "${AUTH_FILE}" ] ; then
echo "Error: Did not find auth file in: ${AUTH_FILE}!"
exit 1
fi
#delete the old secret
kubectl delete secret proxy-auth --namespace=jenkins
# upload the new secret
kubectl create secret generic proxy-auth --from-file="${AUTH_FILE}" --namespace=jenkins
echo "Done."

36
kubernetes/store_ssh_key.sh
View file

@ -1,36 +0,0 @@
#!/bin/bash
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#-------------------------------------------------------------------------------
set -eux
# This scripts creates a new ssh keypair (if it does not exist) and uploads
# it to a kubernetes secret.
#
# You need to manually upload the public key to Github so that you can use
# it for authentication.
LOCAL_SSH_DIR="$HOME/.llvm-premerge-checks/github-ssh"
if [ ! -d "$LOCAL_SSH_DIR" ]; then
mkdir -p "$LOCAL_SSH_DIR"
pushd "$LOCAL_SSH_DIR"
ssh-keygen -b 4096 -t rsa -f "$LOCAL_SSH_DIR/id_rsa" -q -N ""
popd
fi
kubectl create secret generic github-ssh-key --namespace jenkins \
--from-file "$LOCAL_SSH_DIR/id_rsa" \
--from-file "$LOCAL_SSH_DIR/id_rsa.pub"

58
kubernetes/windows-agents/Deployment.yaml
View file

@ -1,58 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: agent-windows-visualstudio2017-ssd
labels:
app: agent-windows-visualstudio2017-ssd
spec:
replicas: 1
selector:
matchLabels:
app: agent-windows-visualstudio2017-ssd
template:
metadata:
labels:
app: agent-windows-visualstudio2017-ssd
spec:
containers:
- name: agent-windows-visualstudio2017-ssd
image: gcr.io/llvm-premerge-checks/agent-windows-jenkins
resources:
limits:
cpu: 14
memory: 20Gi
requests:
cpu: 14
memory: 20Gi
# volumeMounts:
# - name: nfs-pvc
# mountPath: /mnt/nfs
# - name: ssd
# mountPath: /mnt/disks/ssd0
# volumes:
# - name: nfs-pvc
# persistentVolumeClaim:
# claimName: nfs-jenkins
# - name: ssd
# hostPath:
# # directory location on host
# path: /mnt/disks/ssd0
# type: Directory
nodeSelector:
kubernetes.io/os: windows
cloud.google.com/gke-nodepool: windows-pool

19
kubernetes/windows-agents/kustomization.yaml
View file

@ -1,19 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- Deployment.yaml

1
kubernetes/windows_agent_create.sh
View file

@ -21,7 +21,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
ROOT_DIR="$(dirname ${DIR})" ROOT_DIR="$(dirname ${DIR})"
# get config options # get config options
# shellcheck source=../k8s_config
source "${ROOT_DIR}/k8s_config" source "${ROOT_DIR}/k8s_config"
NAME=$1 NAME=$1

1
scripts/clang-tidy-comments.ignore
View file

@ -1,4 +1,5 @@
# Files that are allowed by clang-tidy.ignore but should not receive inline review comments. # Files that are allowed by clang-tidy.ignore but should not receive inline review comments.
# Right now it works in whitelist mode and only some files / directories are whitelisted. # Right now it works in whitelist mode and only some files / directories are whitelisted.
# FIXME: is is used?
* *
!clang-tools-extra/clangd/** !clang-tools-extra/clangd/**

50
scripts/lint.sh
View file

@ -1,50 +0,0 @@
#!/bin/bash
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Runs clang-format
# Inputs: TARGET_DIR, WORKSPACE
# Outputs: ${TARGET_DIR}/clang-format.patch (if there are clang-format findings).
set -eux
echo "Running linters... ====================================="
if (( $# != 2 )); then
echo "Syntax: lint.sh <COMMIT> <OUTPUT_DIR>"
exit 1
fi;
# Commit to diff against
COMMIT="$1"
# output directory for test results
OUTPUT_DIR="$2"
# root directory, where the config files are located
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
if [ ! -f "compile_commands.json" ] ; then
echo "Could not find compile commands.json in $(pwd)"
exit 1
fi
# clang-format
# Let clang format apply patches --diff doesn't produces results in the format we want.
git-clang-format "${COMMIT}"
set +e
git diff -U0 --exit-code --no-prefix | "${DIR}/ignore_diff.py" "${DIR}/clang-format.ignore" > "${OUTPUT_DIR}"/clang-format.patch
set -e
# Revert changes of git-clang-format.
git checkout -- .
# clang-tidy
git diff -U0 --no-prefix "${COMMIT}" | "${DIR}/ignore_diff.py" "${DIR}/clang-tidy.ignore" | clang-tidy-diff -p0 -quiet | sed "/^[[:space:]]*$/d" > "${OUTPUT_DIR}"/clang-tidy.txt
echo "linters completed ======================================"

2
scripts/phabtalk/add_url_artifact.py
View file

@ -21,8 +21,6 @@ import uuid
if __name__ == '__main__': if __name__ == '__main__':
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
# from phabtalk import PhabTalk
# else:
from phabtalk.phabtalk import PhabTalk from phabtalk.phabtalk import PhabTalk

383
scripts/phabtalk/phabtalk.py
View file

@ -12,23 +12,15 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Upload build results to Phabricator. """
Interactions with Phabricator.
"""
As I did not like the Jenkins plugin, we're using this script to upload the
build status, a summary and the test reults to Phabricator."""
import argparse
import os
import re
import socket import socket
import time import time
import urllib
import uuid import uuid
from typing import Optional, List, Dict from typing import Optional, List, Dict
import pathspec
from lxml import etree
from phabricator import Phabricator from phabricator import Phabricator
from enum import IntEnum
class PhabTalk: class PhabTalk:
@ -150,49 +142,6 @@ class PhabTalk:
artifactData=artifact_data)) artifactData=artifact_data))
def _parse_patch(patch) -> List[Dict[str, str]]:
"""Extract the changed lines from `patch` file.
The return value is a list of dictionaries {filename, line, diff}.
Diff must be generated with -U0 (no context lines).
"""
entries = []
lines = []
filename = None
line_number = 0
for line in patch:
match = re.search(r'^(\+\+\+|---) [^/]+/(.*)', line)
if match:
if len(lines) > 0:
entries.append({
'filename': filename,
'diff': ''.join(lines),
'line': line_number,
})
lines = []
filename = match.group(2).rstrip('\r\n')
continue
match = re.search(r'^@@ -(\d+)(,(\d+))? \+(\d+)(,(\d+))?', line)
if match:
if len(lines) > 0:
entries.append({
'filename': filename,
'diff': ''.join(lines),
'line': line_number,
})
lines = []
line_number = int(match.group(1))
continue
if line.startswith('+') or line.startswith('-'):
lines.append(line)
if len(lines) > 0:
entries.append({
'filename': filename,
'diff': ''.join(lines),
'line': line_number,
})
return entries
class Step: class Step:
def __init__(self): def __init__(self):
self.name = '' self.name = ''
@ -236,292 +185,6 @@ class Report:
self.artifacts.append({'dir': dir, 'file': file, 'name': name}) self.artifacts.append({'dir': dir, 'file': file, 'name': name})
class BuildReport:
def __init__(self, args):
# self.args = args
self.ph_id = args.ph_id # type: str
self.diff_id = args.diff_id # type: str
self.test_result_file = args.test_result_file # type: str
self.conduit_token = args.conduit_token # type: str
self.dryrun = args.dryrun # type: bool
self.buildresult = args.buildresult # type: str
self.clang_format_patch = args.clang_format_patch # type: str
self.clang_tidy_result = args.clang_tidy_result # type: str
self.clang_tidy_ignore = args.clang_tidy_ignore # type: str
self.results_dir = args.results_dir # type: str
self.results_url = args.results_url # type: str
self.workspace = args.workspace # type: str
self.failure_messages = args.failures # type: str
self.name = args.name # type: str
self.api = PhabTalk(args.conduit_token, args.host, args.dryrun)
self.revision_id = self.api.get_revision_id(self.diff_id)
self.comments = []
self.success = True
self.working = False
self.unit = [] # type: List
self.lint = {}
self.test_stats = {
'pass': 0,
'fail': 0,
'skip': 0
} # type: Dict[str, int]
def add_lint(self, m):
key = '{}:{}'.format(m['path'], m['line'])
if key not in self.lint:
self.lint[key] = []
self.lint[key].append(m)
def final_report(self):
if self.buildresult is not None:
print('Jenkins result: {}'.format(self.buildresult))
if self.buildresult.lower() == 'success':
pass
elif self.buildresult.lower() == 'null':
self.working = True
else:
self.success = False
else:
self.success = False
try:
self.add_test_results()
except etree.XMLSyntaxError:
# Sometimes we get an incomplete XML file.
# In this case:
# - fail the build (the safe thing to do)
# - continue so the user gets some feedback.
print('Error parsing {}. Invalid XML syntax!'.format(self.test_result_file))
self.success = False
self.add_clang_tidy()
self.add_clang_format()
self.api.update_build_status(self.diff_id, self.ph_id, self.working, self.success, self.lint, self.unit)
self.add_links_to_artifacts()
title = 'Issue with build for {} ({})'.format(self.api.get_revision_id(self.diff_id), self.diff_id)
self.comments.append(
'Pre-merge checks is in beta <a href="https://github.com/google/llvm-premerge-checks/issues/new?assignees'
'=&labels=bug&template=bug_report.md&title={}">report issue</a>.<br/>'
'Please <a href="https://reviews.llvm.org/project/update/78/join/">join beta</a> or '
'<a href="https://github.com/google/llvm-premerge-checks/issues/new?assignees=&labels=enhancement&template'
'=&title=enable%20checks%20for%20{{PATH}}">enable it for your project</a>'.format(
urllib.parse.quote(title)))
with open(os.path.join(self.results_dir, 'summary.html'), 'w') as f:
f.write('<html><head><style>'
'body { font-family: monospace; margin: 16px; }\n'
'.failure {color:red;}\n'
'.success {color:green;}\n'
'</style></head><body>')
f.write('<h1>Build result for diff <a href="https://reviews.llvm.org/{0}">{0}</a> {1} at {2}</h1>'.format(
self.revision_id, self.diff_id, self.name))
if self.failure_messages and len(self.failure_messages) > 0:
for s in self.failure_messages.split('\n'):
f.write('<p class="failure">{}</p>'.format(s))
f.write('<p>' + '</p><p>'.join(self.comments) + '</p>')
f.write('</body></html>')
self.api.add_artifact(self.ph_id, 'summary.html', 'summary ' + self.name, self.results_url)
def add_clang_format(self):
"""Populates results from diff produced by clang format."""
if self.clang_format_patch is None:
return
present = os.path.exists(
os.path.join(self.results_dir, self.clang_format_patch))
if not present:
print('clang-format result {} is not found'.format(self.clang_format_patch))
self.comments.append(section_title('clang-format', False, False))
return
p = os.path.join(self.results_dir, self.clang_format_patch)
if os.stat(p).st_size != 0:
self.api.add_artifact(self.ph_id, self.clang_format_patch, 'clang-format ' + self.name, self.results_url)
diffs = _parse_patch(open(p, 'r'))
success = len(diffs) == 0
for d in diffs:
lines = d['diff'].splitlines(keepends=True)
m = 10 # max number of lines to report.
description = 'please reformat the code\n```\n'
n = len(lines)
cut = n > m + 1
if cut:
lines = lines[:m]
description += ''.join(lines) + '\n```'
if cut:
description += '\n{} diff lines are omitted. See [full diff]({}/{}).'.format(
n - m,
self.results_url,
self.clang_format_patch)
self.add_lint({
'name': 'clang-format',
'severity': 'autofix',
'code': 'clang-format',
'path': d['filename'],
'line': d['line'],
'char': 1,
'description': description,
})
comment = section_title('clang-format', success, present)
if not success:
comment += 'Please format your changes with clang-format by running `git-clang-format HEAD^` or applying ' \
'this <a href="{}">patch</a>.'.format(os.path.basename(self.clang_format_patch))
self.comments.append(comment)
self.success = success and self.success
def add_clang_tidy(self):
if self.clang_tidy_result is None:
return
# Typical message looks like
# [..]/clang/include/clang/AST/DeclCXX.h:3058:20: error: no member named 'LifetimeExtendedTemporary' in 'clang::Decl' [clang-diagnostic-error]
pattern = '^{}/([^:]*):(\\d+):(\\d+): (.*): (.*)'.format(self.workspace)
errors_count = 0
warn_count = 0
inline_comments = 0
present = os.path.exists(
os.path.join(self.results_dir, self.clang_tidy_result))
if not present:
print('clang-tidy result {} is not found'.format(self.clang_tidy_result))
self.comments.append(section_title('clang-tidy', False, False))
return
present = (self.clang_tidy_ignore is not None) and os.path.exists(self.clang_tidy_ignore)
if not present:
print('clang-tidy ignore file {} is not found'.format(self.clang_tidy_ignore))
self.comments.append(section_title('clang-tidy', False, False))
return
p = os.path.join(self.results_dir, self.clang_tidy_result)
add_artifact = False
ignore = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern,
open(self.clang_tidy_ignore, 'r').readlines())
for line in open(p, 'r'):
line = line.strip()
if len(line) == 0 or line == 'No relevant changes found.':
continue
add_artifact = True
match = re.search(pattern, line)
if match:
file_name = match.group(1)
line_pos = match.group(2)
char_pos = match.group(3)
severity = match.group(4)
text = match.group(5)
text += '\n[[{} | not useful]] '.format(
'https://github.com/google/llvm-premerge-checks/blob/master/docs/clang_tidy.md#warning-is-not'
'-useful')
if severity in ['warning', 'error']:
if severity == 'warning':
warn_count += 1
if severity == 'error':
errors_count += 1
if ignore.match_file(file_name):
print('{} is ignored by pattern and no comment will be added'.format(file_name))
else:
inline_comments += 1
self.add_lint({
'name': 'clang-tidy',
'severity': 'warning',
'code': 'clang-tidy',
'path': file_name,
'line': int(line_pos),
'char': int(char_pos),
'description': '{}: {}'.format(severity, text),
})
if add_artifact:
self.api.add_artifact(self.ph_id, self.clang_tidy_result, 'clang-tidy ' + self.name, self.results_url)
success = errors_count + warn_count == 0
comment = section_title('clang-tidy', success, present)
if not success:
comment += 'clang-tidy found <a href="{}">{} errors and {} warnings</a>. ' \
'{} of them are added as review comments <a href="{}">why?</a>.'.format(
self.clang_tidy_result, errors_count, warn_count, inline_comments,
'https://github.com/google/llvm-premerge-checks/blob/master/docs/clang_tidy.md#review-comments')
self.comments.append(comment)
self.success = success and self.success
def add_test_results(self):
"""Populates results from build test results XML.
Only reporting failed tests as the full test suite is too large to upload.
"""
success = True
present = (self.test_result_file is not None) and os.path.exists(
os.path.join(self.results_dir, self.test_result_file))
if not present:
print('Warning: Could not find test results file: {}'.format(self.test_result_file))
self.comments.append(section_title('Unit tests', False, present))
return
root_node = etree.parse(os.path.join(self.results_dir, self.test_result_file))
for test_case in root_node.xpath('//testcase'):
test_result = _test_case_status(test_case)
self.test_stats[test_result] += 1
if test_result == 'fail':
success = False
failure = test_case.find('failure')
test_result = {
'name': test_case.attrib['name'],
'namespace': test_case.attrib['classname'],
'result': test_result,
'duration': float(test_case.attrib['time']),
'details': failure.text
}
self.unit.append(test_result)
comment = section_title('Unit tests', success, True)
comment += '{} tests passed, {} failed and {} were skipped.<br/>'.format(
self.test_stats['pass'],
self.test_stats['fail'],
self.test_stats['skip'],
)
if not success:
comment += 'Failures:<br/>'
for test_case in self.unit:
if test_case['result'] == 'fail':
comment += '{}/{}<br/>'.format(test_case['namespace'], test_case['name'])
self.comments.append(comment)
self.success = success and self.success
def add_links_to_artifacts(self):
"""Comment on a diff, read text from file."""
file_links = []
for f in os.listdir(self.results_dir):
if f == 'summary.html':
continue
if f == 'console-log.txt':
self.api.add_artifact(self.ph_id, f, 'build log ' + self.name, self.results_url)
p = os.path.join(self.results_dir, f)
if not os.path.isfile(p):
continue
if os.stat(p).st_size == 0:
continue
file_links.append('<a href="{0}">{0}</a>'.format(f))
if len(file_links) > 0:
self.comments.append('<a href="./">Build artifacts</a>:<br/>' + '<br/>'.join(file_links))
def _test_case_status(test_case) -> str:
"""Get the status of a test case based on an etree node."""
if test_case.find('failure') is not None:
return 'fail'
if test_case.find('skipped') is not None:
return 'skip'
return 'pass'
def section_title(title: str, ok: bool, present: bool) -> str:
result = 'unknown'
c = ''
if present:
c = 'success' if ok else 'failure'
result = 'pass' if ok else 'fail'
return '{} <span class="{}">{}</span>. '.format(title, c, result)
def _try_call(call): def _try_call(call):
"""Tries to call function several times retrying on socked.timeout.""" """Tries to call function several times retrying on socked.timeout."""
c = 0 c = 0
@ -536,43 +199,3 @@ def _try_call(call):
print('Connection to Pharicator failed, retrying: {}'.format(e)) print('Connection to Pharicator failed, retrying: {}'.format(e))
time.sleep(c * 10) time.sleep(c * 10)
break break
def main():
parser = argparse.ArgumentParser(
description='Write build status back to Phabricator.')
parser.add_argument('ph_id', type=str)
parser.add_argument('diff_id', type=str)
parser.add_argument('--test-result-file', type=str, dest='test_result_file', default='test-results.xml')
parser.add_argument('--conduit-token', type=str, dest='conduit_token', required=True)
parser.add_argument('--host', type=str, dest='host', default="https://reviews.llvm.org/api/",
help="full URL to API with trailing slash, e.g. https://reviews.llvm.org/api/")
parser.add_argument('--dryrun', action='store_true',
help="output results to the console, do not report back to the server")
parser.add_argument('--buildresult', type=str, default=None, choices=['SUCCESS', 'UNSTABLE', 'FAILURE', 'null'])
parser.add_argument('--clang-format-patch', type=str, default=None,
dest='clang_format_patch',
help="path to diff produced by git-clang-format, relative to results-dir")
parser.add_argument('--clang-tidy-result', type=str, default=None,
dest='clang_tidy_result',
help="path to diff produced by git-clang-tidy, relative to results-dir")
parser.add_argument('--clang-tidy-ignore', type=str, default=None,
dest='clang_tidy_ignore',
help="path to file with patters to exclude commenting on for clang-tidy findings")
parser.add_argument('--results-dir', type=str, default=None, required=True,
dest='results_dir',
help="directory of all build artifacts")
parser.add_argument('--results-url', type=str, default=None,
dest='results_url',
help="public URL to access results directory")
parser.add_argument('--workspace', type=str, required=True, help="path to workspace")
parser.add_argument('--failures', type=str, default=None, help="optional failure messages separated by newline")
parser.add_argument('--name', type=str, default='', help="optional name of the build bot")
args = parser.parse_args()
reporter = BuildReport(args)
reporter.final_report()
if __name__ == '__main__':
main()

26
scripts/run_buildkite.ps1
View file

@ -1,26 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
. ${PSScriptRoot}\common.ps1
Write-Output "--- CMake"
& "${PSScriptRoot}\run_cmake.ps1"
Write-Output "--- ninja all"
& "${PSScriptRoot}\run_ninja.ps1" all
Write-Output "--- ninja check-all"
& "${PSScriptRoot}\run_ninja.ps1" check-all
Write-Output "--- done"

39
scripts/run_buildkite.sh
View file

@ -1,39 +0,0 @@
#!/bin/bash
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eux
#folder where this script is stored.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
# dirty workarounds to reuse old scripts...
export WORKSPACE=`pwd`
export TARGET_DIR=/tmp
# create a clean build folder
BUILD_DIR=${WORKSPACE}/build
rm -rf ${BUILD_DIR} || true
mkdir -p ${BUILD_DIR}
echo "--- CMake"
${DIR}/run_cmake.sh
echo "--- ninja all"
${DIR}/run_ninja.sh all
echo "--- ninja check-all"
${DIR}/run_ninja.sh check-all
echo "--- done"

66
scripts/run_cmake.ps1
View file

@ -1,66 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
param (
[Parameter(Mandatory=$false)][string]$projects="default"
)
. ${PSScriptRoot}\common.ps1
# set LLVM_ENABLE_PROJECTS to default value
# if -DetectProjects is set the projects are detected based on the files
# that were modified in the working copy
if ($projects -eq "default") {
# These are the default projects for windows
$LLVM_ENABLE_PROJECTS="clang;clang-tools-extra;libcxx;libc;lld;mlir;libcxxabi"
} elseif ($projects -eq "detect") {
$LLVM_ENABLE_PROJECTS = (git diff HEAD~1 | python ${PSScriptRoot}\choose_projects.py . ) | Out-String
$LLVM_ENABLE_PROJECTS = $LLVM_ENABLE_PROJECTS.replace("`n","").replace("`r","")
if ($LLVM_ENABLE_PROJECTS -eq "") {
Write-Error "Error detecting the affected projects."
exit 1
}
} else {
$LLVM_ENABLE_PROJECTS=$projects
}
Write-Output "Setting LLVM_ENABLE_PROJECTS=${LLVM_ENABLE_PROJECTS}"
# Delete and re-create build folder
Remove-Item build -Recurse -ErrorAction Ignore
New-Item -ItemType Directory -Force -Path build | Out-Null
Push-Location build
# load Vistual Studio environment variables
Invoke-CmdScript "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64
# Make sure we're using the Vistual Studio compiler and linker
$env:CC="cl"
$env:CXX="cl"
$env:LD="link"
# call CMake
$ErrorActionPreference="Continue"
Invoke-Call -ScriptBlock {
cmake ..\llvm -G Ninja -DCMAKE_BUILD_TYPE=Release `
-D LLVM_ENABLE_PROJECTS="${LLVM_ENABLE_PROJECTS}" `
-D LLVM_ENABLE_ASSERTIONS=ON `
-DLLVM_LIT_ARGS="-v --xunit-xml-output test-results.xml" `
-D LLVM_ENABLE_DIA_SDK=OFF
}
# LLVM_ENABLE_DIA_SDK=OFF is a workaround to make the tests pass.
# see https://bugs.llvm.org/show_bug.cgi?id=44151
Pop-Location

41
scripts/run_cmake.sh
View file

@ -1,41 +0,0 @@
#!/bin/bash
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eux
# Runs Cmake.
# Inputs: CCACHE_DIR, WORKSPACE, TARGET_DIR; $WORKSPACE/build must exist.
# Outputs: $TARGET_DIR/CMakeCache.txt, $WORKSPACE/compile_commands.json (symlink).
echo "Running CMake... ======================================"
export CC=clang
export CXX=clang++
export LD=LLD
cd "$WORKSPACE"/build
set +e
cmake -GNinja ../llvm -DCMAKE_BUILD_TYPE=Release -D LLVM_ENABLE_LLD=ON \
-D LLVM_ENABLE_PROJECTS="clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;flang" \
-D LLVM_CCACHE_BUILD=ON -D LLVM_CCACHE_DIR="${CCACHE_DIR}" -D LLVM_CCACHE_MAXSIZE=20G \
-D LLVM_ENABLE_ASSERTIONS=ON -DCMAKE_CXX_FLAGS=-gmlt \
-DLLVM_LIT_ARGS="-v --xunit-xml-output ${WORKSPACE}/build/test-results.xml"
RETURN_CODE="${PIPESTATUS[0]}"
set -e
rm -f "$WORKSPACE/compile_commands.json"
ln -s "$WORKSPACE"/build/compile_commands.json "$WORKSPACE"
cp CMakeCache.txt ${TARGET_DIR}
echo "CMake completed ======================================"
exit "${RETURN_CODE}"

31
scripts/run_ninja.ps1
View file

@ -1,31 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
param(
[Parameter(Mandatory=$true)][string]$target
)
. ${PSScriptRoot}\common.ps1
# cd into build folder
Push-Location build
# load Visual Studio environment variables
Invoke-CmdScript "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64
# call ninja
$PSDefaultParameterValues['*:ErrorAction']='Continue'
Invoke-Call -ScriptBlock {ninja $target}
Pop-Location

42
scripts/run_ninja.sh
View file

@ -1,42 +0,0 @@
#!/bin/bash
# Copyright 2019 Google LLC
#
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://llvm.org/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eu
# Runs ninja
# Inputs: TARGET_DIR, WORKSPACE.
# Outputs: $TARGET_DIR/test_results.xml
CMD=$1
echo "Running ninja ${CMD}... ====================================="
ulimit -n 8192
cd "${WORKSPACE}/build"
set +e
ninja ${CMD}
RETURN_CODE="$?"
set -e
echo "ninja ${CMD} completed ======================================"
if test -f "test-results.xml" ; then
echo "copying test_results.xml to ${TARGET_DIR}"
# wait for file?
sleep 10s
du "test-results.xml"
cp test-results.xml "${TARGET_DIR}"
sleep 10s
fi
exit ${RETURN_CODE}

53
scripts/windows_agent_start_jenkins.ps1
View file

@ -1,53 +0,0 @@
# Copyright 2019 Google LLC
# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# https://llvm.org/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Pull and start the Docker container for a Windows agent.
# To setup a Windows agent see docs/playbooks.md
param(
[string]$version = "latest",
[switch]$testing = $false
)
$NAME="agent-windows-jenkins"
$IMAGE="gcr.io/llvm-premerge-checks/${NAME}:${version}"
Write-Output "Authenticating docker..."
Write-Output "y`n" | gcloud auth configure-docker
Write-Output "Pulling new image..."
docker pull ${IMAGE}
Write-Output "Stopping old container..."
docker stop ${NAME}
docker rm ${NAME}
Write-Output "Starting container..."
if (${testing}) {
docker run -it `
-v D:\:C:\ws `
-v C:\credentials:C:\credentials `
-e PARENT_HOSTNAME=$env:computername `
--restart unless-stopped `
--name ${NAME} `
${IMAGE} powershell
} else {
docker run -d `
-v D:\:C:\ws `
-v C:\credentials:C:\credentials `
-e PARENT_HOSTNAME=$env:computername `
--restart unless-stopped `
--name ${NAME} `
${IMAGE}
}