diff --git a/Jenkins/BETA-apply-patch/Jenkinsfile b/Jenkins/BETA-apply-patch/Jenkinsfile deleted file mode 100644 index c8bec71..0000000 --- a/Jenkins/BETA-apply-patch/Jenkinsfile +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://llvm.org/LICENSE.txt -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -def success = true -def failure_message = "" - -pipeline { - agent { label 'linux' } - parameters { - string(name: 'DIFF_ID') - string(name: 'PHID') - string(name: 'REV_ID') - } - environment { - CONDUIT_TOKEN = credentials('phabricator-conduit-token') - PHABRICATOR_HOST = 'https://reviews.llvm.org' - LLVM_DIR = "${WORKSPACE}/llvm-project" - SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts" - RESULT_DIR = "${WORKSPACE}/results" - PHAB_LOG = "${RESULT_DIR}/.phabricator-comment" - MY_BUILD_ID = "${JOB_BASE_NAME}-${BUILD_NUMBER}" - TARGET_DIR = "/mnt/nfs/results/${MY_BUILD_ID}" - RESULT_URL = "http://results.llvm-merge-guard.org/${MY_BUILD_ID}" - } - options { - timeout(time:10, unit:'MINUTES') - } - stages { - stage("build info"){ - steps { - echo "Building diff ${DIFF_ID} with PHID ${PHID} for Revision ${REV_ID}" - script { - currentBuild.displayName += " D${REV_ID}" - currentBuild.description = "D${REV_ID}" - } - } - } - stage("git checkout"){ - steps { - dir("${LLVM_DIR}"){ - git url: 'git@github.com:llvm-premerge-tests/llvm-project.git' - } - dir("llvm-premerge-checks") - { - git url: 'https://github.com/google/llvm-premerge-checks.git' - } - sh """ - rm -rf ${RESULT_DIR} - mkdir -p ${RESULT_DIR} - """ - } - } - stage('arc patch'){ - steps { - dir("${LLVM_DIR}"){ - sh """ - ${SCRIPT_DIR}/phabtalk/apply_patch2.py ${DIFF_ID} \ - --token ${CONDUIT_TOKEN} \ - --url ${PHABRICATOR_HOST} \ - --push-branch - """ - } - } - } - } - post { - always { - dir("${RESULT_DIR}") { - // copy console log to result folder - sh "wget -qO console-log.txt http://jenkins-ui.jenkins.svc.cluster.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText" - sh "mkdir ${TARGET_DIR}" - sh "cp * ${TARGET_DIR}" - } - - sh """${SCRIPT_DIR}/phabtalk/phabtalk.py "${PHID}" "${DIFF_ID}" \ - --workspace "${LLVM_DIR}" \ - --conduit-token "${CONDUIT_TOKEN}" \ - --host "${PHABRICATOR_HOST}/api/" \ - --buildresult ${currentBuild.result} \ - --results-dir "${TARGET_DIR}" \ - --results-url "${RESULT_URL}" \ - --failures "Applying patch failed" \ - --name "apply patch" - """ - } - } -} diff --git a/Jenkins/BETA-phabricator-linux/Jenkinsfile b/Jenkins/BETA-phabricator-linux/Jenkinsfile deleted file mode 100644 index eadfb31..0000000 --- a/Jenkins/BETA-phabricator-linux/Jenkinsfile +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://llvm.org/LICENSE.txt -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -def success = true -def failure_message = "" - -pipeline { - agent { label 'linux' } - parameters { - string(name: 'DIFF_ID') - string(name: 'PHID') - string(name: 'REV_ID') - } - environment { - CONDUIT_TOKEN = credentials('phabricator-conduit-token') - PHABRICATOR_HOST = 'https://reviews.llvm.org' - LLVM_DIR = "${WORKSPACE}/llvm-project" - SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts" - RESULT_DIR = "${WORKSPACE}/results" - PHAB_LOG = "${RESULT_DIR}/.phabricator-comment" - MY_BUILD_ID = "${JOB_BASE_NAME}-${BUILD_NUMBER}" - TARGET_DIR = "/mnt/nfs/results/${MY_BUILD_ID}" - RESULT_URL = "http://results.llvm-merge-guard.org/${MY_BUILD_ID}" - } - options { - timeout(time:2, unit:'HOURS') - } - stages { - stage("build info"){ - steps { - echo "Building diff ${DIFF_ID} with PHID ${PHID} for Revision ${REV_ID}" - script { - currentBuild.displayName += " D${REV_ID}" - currentBuild.description = "D${REV_ID}" - } - sh """ - rm -rf ${RESULT_DIR} - mkdir -p ${RESULT_DIR} - """ - // Report versions of the installed packages. - sh """ - echo Versions of various tools: > ${RESULT_DIR}/package_version.log - clang --version >> ${RESULT_DIR}/package_version.log - clang-tidy --version >> ${RESULT_DIR}/package_version.log - ld.lld --version >> ${RESULT_DIR}/package_version.log - dpkg -l >> ${RESULT_DIR}/package_version.log - """ - } - } - stage("git checkout") - { - steps { - dir("${LLVM_DIR}") - { - git url: 'git@github.com:llvm-premerge-tests/llvm-project.git', branch: "phab-diff-${DIFF_ID}" - sh 'git clean -fdx' - sh 'git show -s' - } - dir("llvm-premerge-checks") - { - git url: 'https://github.com/google/llvm-premerge-checks.git' - } - } - } - stage('CMake') { - steps { - dir("${LLVM_DIR}") - { - script { - success = false - failure_message = "Failed to run cmake" - } - sh "${SCRIPT_DIR}/run_cmake.py detect" - script { - success = true - failure_message = "" - } - } - } - } - stage('ninja all') { - steps { - dir("${LLVM_DIR}") - { - script { - try { - sh(script: "${SCRIPT_DIR}/run_ninja.py all") - } catch (e) { - success = false; - failure_message = "'ninja all' failed" // append as build might already be broken - echo e.toString() - } - } - } - } - } - stage('ninja check-all') { - steps { - dir("${LLVM_DIR}") - { - script { - if (success) { - // only run tests if build has passed, see #176 - try { - sh(script: "${SCRIPT_DIR}/run_ninja.py check-all") - } catch (e) { - success = false; - failure_message += "\n'ninja check-all' failed" // append as build might already be broken - echo e.toString() - } - } - } - } - } - } - stage('linters') { - steps { - dir("${LLVM_DIR}") - { - script { - try { - sh(script: "${SCRIPT_DIR}/lint.sh HEAD~1 ${RESULT_DIR}") - } catch (e) { - success = false; - failure_message += "\nFailed to run linters" // append as build might already be broken - echo e.toString() - } - } - } - } - } - } - post { - always { - script { - if (success) { - currentBuild.result = "SUCCESS" - } else { - currentBuild.result = "FAILURE" - } - } - echo "Console log is available at ${RESULT_URL}" - dir("${RESULT_DIR}") { - // copy console log to result folder - sh "wget -qO console-log.txt http://jenkins-ui.jenkins.svc.cluster.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText" - // keep a copy of the answer from Phabricator for debugging - // TODO: move all file copy operations here - sh """ - set -eu - cp ${LLVM_DIR}/build/CMakeCache.txt . || : - cp ${LLVM_DIR}/build/test-results.xml . || : - mkdir -p ${TARGET_DIR} - cp * ${TARGET_DIR} - """ - } - /// send results to Phabricator - sh """${SCRIPT_DIR}/phabtalk/phabtalk.py "${PHID}" "${DIFF_ID}" \ - --workspace "${WORKSPACE}" \ - --conduit-token "${CONDUIT_TOKEN}" \ - --test-result-file "${RESULT_DIR}/test-results.xml" \ - --host "${PHABRICATOR_HOST}/api/" \ - --buildresult ${currentBuild.result} \ - --clang-format-patch "${RESULT_DIR}/clang-format.patch" \ - --clang-tidy-result "${RESULT_DIR}/clang-tidy.txt" \ - --clang-tidy-ignore "${SCRIPT_DIR}/clang-tidy-comments.ignore" \ - --results-dir "${TARGET_DIR}" \ - --results-url "${RESULT_URL}" \ - --failures "${failure_message}" \ - --name "linux" - """ - } - } -} diff --git a/Jenkins/BETA-phabricator-windows/Jenkinsfile b/Jenkins/BETA-phabricator-windows/Jenkinsfile deleted file mode 100644 index 42b59bd..0000000 --- a/Jenkins/BETA-phabricator-windows/Jenkinsfile +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://llvm.org/LICENSE.txt -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -def success = true - -pipeline { - agent { - node { - label 'windows' - // use custom workspace folder to shorten paths - customWorkspace "C:\\ws\\beta" - } - } - parameters { - string(name: 'DIFF_ID') - string(name: 'PHID') - string(name: 'REV_ID') - } - environment { - CONDUIT_TOKEN = credentials('phabricator-conduit-token') - PHABRICATOR_HOST = 'https://reviews.llvm.org' - PHAB_LOG = "${WORKSPACE}/build/.phabricator-comment" - MY_BUILD_ID = "${JOB_BASE_NAME}-${BUILD_NUMBER}" - RESULT_URL = "https://storage.googleapis.com/llvm-premerge-checks/results/${MY_BUILD_ID}" - SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts" - // store all build results here, will be uploaded to GCS later - RESULT_DIR = "${WORKSPACE}\\results" - LLVM_DIR = "${WORKSPACE}\\llvm-project" - // enable sccache for this build. Comment out the line to disable it - SCCACHE_DIR = "C:\\ws\\sccache" - } - options { - timeout(time:2, unit:'HOURS') - } - stages { - stage("build info"){ - steps { - echo "Building diff ${DIFF_ID} with PHID ${PHID} for Revision ${REV_ID}" - script { - currentBuild.displayName += " D${REV_ID}" - currentBuild.description = "D${REV_ID}" - } - script { - success = true - failure_message = "" - } - } - } - stage("git checkout"){ - steps { - echo "getting llvm-premerge-checks... " - dir("llvm-premerge-checks") - { - git url: 'https://github.com/google/llvm-premerge-checks.git' - } - echo "getting llvm-project... " - dir("llvm-project") - { - git url: 'https://github.com/llvm-premerge-tests/llvm-project.git', branch: "phab-diff-${DIFF_ID}" - powershell 'git clean -fdx' - powershell 'git show -s' - } - // create ${RESULT_DIR} - powershell """ - Remove-Item ${RESULT_DIR} -Recurse -ErrorAction Ignore - New-Item -ItemType Directory -Force -Path ${RESULT_DIR} | Out-Null - """ - } - } - stage('CMake') { - steps { - dir("${LLVM_DIR}"){ - powershell "python ${SCRIPT_DIR}/run_cmake.py detect" - } - } - post { - failure { - script { - success = false - failure_message = "Failed to run CMake" - } - } - } - } - stage('ninja all') { - steps { - dir("${LLVM_DIR}"){ - powershell "python ${SCRIPT_DIR}/run_ninja.py all" - } - } - post { - failure { - script { - success = false - failure_message = "Failed to run ninja all" - } - } - } - } - stage('ninja check-all') { - steps { - dir("${LLVM_DIR}"){ - powershell "python ${SCRIPT_DIR}/run_ninja.py check-all" - } - } - post { - failure { - script { - success = false - failure_message = "Failed to run ninja check-all" - } - } - } - } - } - post { - always { - script { - if (success) { - currentBuild.result = "SUCCESS" - } else { - currentBuild.result = "FAILURE" - } - } - echo "Uploading logs to ${RESULT_URL} ..." - dir("${RESULT_DIR}") { - // gather all result files in a folder - powershell """ - \$ErrorActionPreference = 'Continue' - Write-Host "Getting the console log..." - Invoke-WebRequest -OutFile console-log.txt -uri "http://jenkins.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText" -ErrorAction "Continue" - - Write-Host "Copying build artefacts..." - Copy-Item "${LLVM_DIR}\\choose_projects.log" - Copy-Item "${LLVM_DIR}\\build\\CMakeCache.txt" - Copy-Item "${LLVM_DIR}\\build\\test-results.xml" - """ - } - /// send results to Phabricator - echo "Sending build feedback to Phabricator..." - bat """ - python ${SCRIPT_DIR}/phabtalk/phabtalk.py "${PHID}" "${DIFF_ID}" ^ - --workspace "${LLVM_DIR}" ^ - --conduit-token "${CONDUIT_TOKEN}" ^ - --test-result-file "test-results.xml" ^ - --host "${PHABRICATOR_HOST}/api/" ^ - --results-dir "${RESULT_DIR}" ^ - --results-url "${RESULT_URL}" ^ - --failures "${failure_message}" ^ - --buildresult ${currentBuild.result} ^ - --name "windows" - """ - dir("${RESULT_DIR}") { - // upload results to - // Google Cloud Storage - powershell """ - Write-Host "Uploading results to GCS..." - \$ErrorActionPreference = 'Continue' - gsutil cp *.* gs://llvm-premerge-checks/results/${MY_BUILD_ID}/ - Write-Host "Done." - """ - } - echo "Done." - } - } -} \ No newline at end of file diff --git a/Jenkins/apply-patch/Jenkinsfile b/Jenkins/apply-patch/Jenkinsfile deleted file mode 100644 index c8bec71..0000000 --- a/Jenkins/apply-patch/Jenkinsfile +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://llvm.org/LICENSE.txt -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -def success = true -def failure_message = "" - -pipeline { - agent { label 'linux' } - parameters { - string(name: 'DIFF_ID') - string(name: 'PHID') - string(name: 'REV_ID') - } - environment { - CONDUIT_TOKEN = credentials('phabricator-conduit-token') - PHABRICATOR_HOST = 'https://reviews.llvm.org' - LLVM_DIR = "${WORKSPACE}/llvm-project" - SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts" - RESULT_DIR = "${WORKSPACE}/results" - PHAB_LOG = "${RESULT_DIR}/.phabricator-comment" - MY_BUILD_ID = "${JOB_BASE_NAME}-${BUILD_NUMBER}" - TARGET_DIR = "/mnt/nfs/results/${MY_BUILD_ID}" - RESULT_URL = "http://results.llvm-merge-guard.org/${MY_BUILD_ID}" - } - options { - timeout(time:10, unit:'MINUTES') - } - stages { - stage("build info"){ - steps { - echo "Building diff ${DIFF_ID} with PHID ${PHID} for Revision ${REV_ID}" - script { - currentBuild.displayName += " D${REV_ID}" - currentBuild.description = "D${REV_ID}" - } - } - } - stage("git checkout"){ - steps { - dir("${LLVM_DIR}"){ - git url: 'git@github.com:llvm-premerge-tests/llvm-project.git' - } - dir("llvm-premerge-checks") - { - git url: 'https://github.com/google/llvm-premerge-checks.git' - } - sh """ - rm -rf ${RESULT_DIR} - mkdir -p ${RESULT_DIR} - """ - } - } - stage('arc patch'){ - steps { - dir("${LLVM_DIR}"){ - sh """ - ${SCRIPT_DIR}/phabtalk/apply_patch2.py ${DIFF_ID} \ - --token ${CONDUIT_TOKEN} \ - --url ${PHABRICATOR_HOST} \ - --push-branch - """ - } - } - } - } - post { - always { - dir("${RESULT_DIR}") { - // copy console log to result folder - sh "wget -qO console-log.txt http://jenkins-ui.jenkins.svc.cluster.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText" - sh "mkdir ${TARGET_DIR}" - sh "cp * ${TARGET_DIR}" - } - - sh """${SCRIPT_DIR}/phabtalk/phabtalk.py "${PHID}" "${DIFF_ID}" \ - --workspace "${LLVM_DIR}" \ - --conduit-token "${CONDUIT_TOKEN}" \ - --host "${PHABRICATOR_HOST}/api/" \ - --buildresult ${currentBuild.result} \ - --results-dir "${TARGET_DIR}" \ - --results-url "${RESULT_URL}" \ - --failures "Applying patch failed" \ - --name "apply patch" - """ - } - } -} diff --git a/Jenkins/daily-cleanup/Jenkinsfile b/Jenkins/daily-cleanup/Jenkinsfile deleted file mode 100644 index a2c7f4a..0000000 --- a/Jenkins/daily-cleanup/Jenkinsfile +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://llvm.org/LICENSE.txt -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - - -/* Pipeline executed daily to do cleanup work */ - -def success = true -def failure_message = "" - -pipeline { - agent { label 'linux' } - triggers { - cron('@daily') - } - environment { - LLVM_DIR = "${WORKSPACE}/llvm-project" - SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts" - } - options { - timeout(time:30, unit:'MINUTES') - } - stages { - stage("git checkout"){ - steps { - dir("${LLVM_DIR}"){ - git url: 'git@github.com:llvm-premerge-tests/llvm-project.git' - } - dir("llvm-premerge-checks") - { - git url: 'https://github.com/google/llvm-premerge-checks.git' - } - } - } - stage('cleanup repo'){ - // delete old branches after 30 days - steps { - dir("${LLVM_DIR}"){ - sh """ - python ${SCRIPT_DIR}/cleanup_branches.py --days 30 --pattern "phab-diff-.*" - """ - } - } - } - } -} diff --git a/Jenkins/master-linux-pipeline/Jenkinsfile b/Jenkins/master-linux-pipeline/Jenkinsfile deleted file mode 100644 index 292d215..0000000 --- a/Jenkins/master-linux-pipeline/Jenkinsfile +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://llvm.org/LICENSE.txt -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pipeline { - agent { label 'linux' } - triggers { - pollSCM 'H H/2 * * *' - } - environment { - BUILD_ID="${JOB_BASE_NAME}-${BUILD_NUMBER}" - TARGET_DIR="/mnt/nfs/results/${BUILD_ID}" - SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts" - } - options { - // set build timeout - timeout(time:2, unit:'HOURS') - } - stages { - stage("git checkout"){ - steps { - git url: 'https://github.com/llvm/llvm-project.git' - sh 'git clean -fdx' - sh 'mkdir -p llvm-premerge-checks' - dir("llvm-premerge-checks") - { - git url: 'https://github.com/google/llvm-premerge-checks.git' - } - sh 'rm -rf build || true' - sh 'mkdir -p build' - sh 'mkdir -p "${TARGET_DIR}"' - } - } - stage('CMake') { - steps { - sh "${SCRIPT_DIR}/run_cmake.py" - } - } - stage('ninja all') { - steps { - sh "${SCRIPT_DIR}/run_ninja.py all" - } - } - stage('ninja check-all') { - steps { - sh "${SCRIPT_DIR}/run_ninja.py check-all" - } - } - } - post { - always { - echo "Console log is available at https://results.llvm-merge-guard.org/${BUILD_ID}" - dir("${env.TARGET_DIR}") { - // copy console log to result folder - sh "wget -qO console-log.txt http://jenkins-ui.jenkins.svc.cluster.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText" - // TODO: move all file copy operations here - // junit 'build/*.xml' - } - } - } -} \ No newline at end of file diff --git a/Jenkins/master-windows-pipeline/Jenkinsfile b/Jenkins/master-windows-pipeline/Jenkinsfile deleted file mode 100644 index 7ccd87b..0000000 --- a/Jenkins/master-windows-pipeline/Jenkinsfile +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://llvm.org/LICENSE.txt -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pipeline { - agent { - node { - label 'windows' - // use custom workspace folder to shorten paths - customWorkspace "C:\\ws\\master" - } - } - - triggers { - pollSCM 'H H/4 * * *' - } - - options { - // enable timestaps for getting execution times - timestamps () - // set build timeout - timeout(time:2, unit:'HOURS') - } - - environment { - MY_BUILD_ID = "${JOB_BASE_NAME}-${BUILD_NUMBER}" - SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts" - RESULT_DIR = "${WORKSPACE}\\results" - LLVM_DIR = "${WORKSPACE}\\llvm-project" - // enable sccache for this build. Comment out the line to disable it - SCCACHE_DIR = "C:\\ws\\sccache" - RESULT_URL = "https://storage.googleapis.com/llvm-premerge-checks/results/${MY_BUILD_ID}" - } - - stages { - stage("git checkout"){ - steps { - echo "getting llvm-premerge-checks... " - dir("llvm-premerge-checks") - { - git url: 'https://github.com/google/llvm-premerge-checks.git' - } - echo "getting llvm-project... " - dir("llvm-project") - { - git url: 'https://github.com/llvm/llvm-project.git' - } - powershell "New-Item -ItemType Directory -Force -Path ${RESULT_DIR}" - } - } - stage('CMake') { - steps { - dir("${LLVM_DIR}"){ - powershell "python ${SCRIPT_DIR}/run_cmake.py" - } - } - } - stage('ninja all') { - steps { - dir("${LLVM_DIR}"){ - powershell "python ${SCRIPT_DIR}/run_ninja.py all" - } - } - } - stage('ninja check-all') { - steps { - dir("${LLVM_DIR}"){ - powershell "python ${SCRIPT_DIR}/run_ninja.py check-all" - } - } - } - } - post { - always { - echo "Logs uploaded to ${RESUlT_URL}" - dir("${env.RESULT_DIR}") { - // gather all result files in a folder, then upload everything to - // Google Cloud Storage - powershell """ - # get the console log - Invoke-WebRequest -OutFile console-log.txt -uri "http://jenkins.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText" -ErrorAction "Continue" - - Copy-Item "${LLVM_DIR}\\build\\CMakeCache.txt" - Copy-Item "${LLVM_DIR}\\build\\test-results.xml" - - # upload files - \$ErrorActionPreference = 'SilentlyContinue' - gsutil cp -Z *.* gs://llvm-premerge-checks/results/${MY_BUILD_ID}/ - """ - } - // doesn't find junit results, not sure why... - // junit "${LLVM_DIR}\\build\\test-results.xml" - } - } - -} \ No newline at end of file diff --git a/Jenkins/phabricator-linux/Jenkinsfile b/Jenkins/phabricator-linux/Jenkinsfile deleted file mode 100644 index 32301ba..0000000 --- a/Jenkins/phabricator-linux/Jenkinsfile +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://llvm.org/LICENSE.txt -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -def success = true -def failure_message = "" - -pipeline { - agent { label 'linux' } - parameters { - string(name: 'DIFF_ID') - string(name: 'PHID') - string(name: 'REV_ID') - } - environment { - CONDUIT_TOKEN = credentials('phabricator-conduit-token') - PHABRICATOR_HOST = 'https://reviews.llvm.org' - LLVM_DIR = "${WORKSPACE}/llvm-project" - SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts" - RESULT_DIR = "${WORKSPACE}/results" - PHAB_LOG = "${RESULT_DIR}/.phabricator-comment" - MY_BUILD_ID = "${JOB_BASE_NAME}-${BUILD_NUMBER}" - TARGET_DIR = "/mnt/nfs/results/${MY_BUILD_ID}" - RESULT_URL = "http://results.llvm-merge-guard.org/${MY_BUILD_ID}" - } - options { - timeout(time:2, unit:'HOURS') - } - stages { - stage("build info"){ - steps { - echo "Building diff ${DIFF_ID} with PHID ${PHID} for Revision ${REV_ID}" - script { - currentBuild.displayName += " D${REV_ID}" - currentBuild.description = "D${REV_ID}" - } - sh """ - rm -rf ${RESULT_DIR} - mkdir -p ${RESULT_DIR} - """ - // Report versions of the installed packages. - sh """ - echo Versions of various tools: > ${RESULT_DIR}/package_version.log - clang --version >> ${RESULT_DIR}/package_version.log - clang-tidy --version >> ${RESULT_DIR}/package_version.log - ld.lld --version >> ${RESULT_DIR}/package_version.log - dpkg -l >> ${RESULT_DIR}/package_version.log - """ - } - } - stage("git checkout") - { - steps { - dir("${LLVM_DIR}") - { - git url: 'git@github.com:llvm-premerge-tests/llvm-project.git', branch: "phab-diff-${DIFF_ID}" - sh 'git clean -fdx' - sh 'git show -s' - } - dir("llvm-premerge-checks") - { - git url: 'https://github.com/google/llvm-premerge-checks.git' - } - } - } - stage('CMake') { - steps { - dir("${LLVM_DIR}") - { - script { - success = false - failure_message = "Failed to run cmake" - } - sh "${SCRIPT_DIR}/run_cmake.py detect" - script { - success = true - failure_message = "" - } - } - } - } - stage('ninja all') { - steps { - dir("${LLVM_DIR}") - { - script { - try { - sh(script: "${SCRIPT_DIR}/run_ninja.py all") - } catch (e) { - success = false; - failure_message = "'ninja all' failed" // append as build might already be broken - echo e.toString() - } - } - } - } - } - stage('ninja check-all') { - steps { - dir("${LLVM_DIR}") - { - script { - try { - sh(script: "${SCRIPT_DIR}/run_ninja.py check-all") - } catch (e) { - success = false; - failure_message += "\n'ninja check-all' failed" // append as build might already be broken - echo e.toString() - } - } - } - } - } - stage('linters') { - steps { - dir("${LLVM_DIR}") - { - script { - try { - sh(script: "${SCRIPT_DIR}/lint.sh HEAD~1 ${RESULT_DIR}") - } catch (e) { - success = false; - failure_message += "\nFailed to run linters" // append as build might already be broken - echo e.toString() - } - } - } - } - } - } - post { - always { - script { - if (success) { - currentBuild.result = "SUCCESS" - } else { - currentBuild.result = "FAILURE" - } - } - echo "Console log is available at ${RESULT_URL}" - dir("${RESULT_DIR}") { - // copy console log to result folder - sh "wget -qO console-log.txt http://jenkins-ui.jenkins.svc.cluster.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText" - // keep a copy of the answer from Phabricator for debugging - // TODO: move all file copy operations here - sh """ - set -eu - cp ${LLVM_DIR}/build/CMakeCache.txt . || : - cp ${LLVM_DIR}/build/test-results.xml . || : - mkdir -p ${TARGET_DIR} - cp * ${TARGET_DIR} - """ - } - /// send results to Phabricator - sh """${SCRIPT_DIR}/phabtalk/phabtalk.py "${PHID}" "${DIFF_ID}" \ - --workspace "${WORKSPACE}" \ - --conduit-token "${CONDUIT_TOKEN}" \ - --test-result-file "${RESULT_DIR}/test-results.xml" \ - --host "${PHABRICATOR_HOST}/api/" \ - --buildresult ${currentBuild.result} \ - --clang-format-patch "${RESULT_DIR}/clang-format.patch" \ - --clang-tidy-result "${RESULT_DIR}/clang-tidy.txt" \ - --clang-tidy-ignore "${SCRIPT_DIR}/clang-tidy-comments.ignore" \ - --results-dir "${TARGET_DIR}" \ - --results-url "${RESULT_URL}" \ - --failures "${failure_message}" \ - --name "linux" - """ - } - } -} diff --git a/Jenkins/phabricator-windows/Jenkinsfile b/Jenkins/phabricator-windows/Jenkinsfile deleted file mode 100644 index 313b2ec..0000000 --- a/Jenkins/phabricator-windows/Jenkinsfile +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://llvm.org/LICENSE.txt -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -def success = true - -pipeline { - agent { - node { - label 'windows' - // use custom workspace folder to shorten paths - customWorkspace "C:\\ws\\prod" - } - } - parameters { - string(name: 'DIFF_ID') - string(name: 'PHID') - string(name: 'REV_ID') - } - environment { - CONDUIT_TOKEN = credentials('phabricator-conduit-token') - PHABRICATOR_HOST = 'https://reviews.llvm.org' - PHAB_LOG = "${WORKSPACE}/build/.phabricator-comment" - MY_BUILD_ID = "${JOB_BASE_NAME}-${BUILD_NUMBER}" - RESULT_URL = "https://storage.googleapis.com/llvm-premerge-checks/results/${MY_BUILD_ID}" - SCRIPT_DIR = "${WORKSPACE}/llvm-premerge-checks/scripts" - // store all build results here, will be uploaded to GCS later - RESULT_DIR = "${WORKSPACE}\\results" - LLVM_DIR = "${WORKSPACE}\\llvm-project" - // enable sccache for this build. Comment out the line to disable it - SCCACHE_DIR = "C:\\ws\\sccache" - } - options { - timeout(time:2, unit:'HOURS') - } - stages { - stage("build info"){ - steps { - echo "Building diff ${DIFF_ID} with PHID ${PHID} for Revision ${REV_ID}" - script { - currentBuild.displayName += " D${REV_ID}" - currentBuild.description = "D${REV_ID}" - } - script { - success = true - failure_message = "" - } - } - } - stage("git checkout"){ - steps { - echo "getting llvm-premerge-checks... " - dir("llvm-premerge-checks") - { - git url: 'https://github.com/google/llvm-premerge-checks.git' - } - echo "getting llvm-project... " - dir("llvm-project") - { - git url: 'https://github.com/llvm-premerge-tests/llvm-project.git', branch: "phab-diff-${DIFF_ID}" - powershell 'git clean -fdx' - powershell 'git show -s' - } - // create ${RESULT_DIR} - powershell """ - Remove-Item ${RESULT_DIR} -Recurse -ErrorAction Ignore - New-Item -ItemType Directory -Force -Path ${RESULT_DIR} | Out-Null - """ - } - } - stage('CMake') { - steps { - dir("${LLVM_DIR}"){ - powershell "python ${SCRIPT_DIR}/run_cmake.py detect" - } - } - post { - failure { - script { - success = false - failure_message = "Failed to run CMake" - } - } - } - } - stage('ninja all') { - steps { - dir("${LLVM_DIR}"){ - powershell "python ${SCRIPT_DIR}/run_ninja.py all" - } - } - post { - failure { - script { - success = false - failure_message = "Failed to run ninja all" - } - } - } - } - stage('ninja check-all') { - steps { - dir("${LLVM_DIR}"){ - powershell "python ${SCRIPT_DIR}/run_ninja.py check-all" - } - } - post { - failure { - script { - success = false - failure_message = "Failed to run ninja check-all" - } - } - } - } - } - post { - always { - script { - if (success) { - currentBuild.result = "SUCCESS" - } else { - currentBuild.result = "FAILURE" - } - } - echo "Uploading logs to ${RESULT_URL} ..." - dir("${RESULT_DIR}") { - // gather all result files in a folder - powershell """ - \$ErrorActionPreference = 'Continue' - Write-Host "Getting the console log..." - Invoke-WebRequest -OutFile console-log.txt -uri "http://jenkins.local:8080/job/${JOB_BASE_NAME}/${BUILD_NUMBER}/consoleText" -ErrorAction "Continue" - - Write-Host "Copying build artefacts..." - Copy-Item "${LLVM_DIR}\\choose_projects.log" - Copy-Item "${LLVM_DIR}\\build\\CMakeCache.txt" - Copy-Item "${LLVM_DIR}\\build\\test-results.xml" - """ - } - /// send results to Phabricator - echo "Sending build feedback to Phabricator..." - bat """ - python ${SCRIPT_DIR}/phabtalk/phabtalk.py "${PHID}" "${DIFF_ID}" ^ - --workspace "${LLVM_DIR}" ^ - --conduit-token "${CONDUIT_TOKEN}" ^ - --test-result-file "test-results.xml" ^ - --host "${PHABRICATOR_HOST}/api/" ^ - --results-dir "${RESULT_DIR}" ^ - --results-url "${RESULT_URL}" ^ - --failures "${failure_message}" ^ - --buildresult ${currentBuild.result} ^ - --name "windows" - """ - dir("${RESULT_DIR}") { - // upload results to - // Google Cloud Storage - powershell """ - Write-Host "Uploading results to GCS..." - \$ErrorActionPreference = 'Continue' - gsutil cp *.* gs://llvm-premerge-checks/results/${MY_BUILD_ID}/ - Write-Host "Done." - """ - } - echo "Done." - } - } -} \ No newline at end of file diff --git a/containers/agent-debian-testing-buildkite/Dockerfile b/containers/agent-debian-testing-buildkite/Dockerfile deleted file mode 100644 index 6c025f8..0000000 --- a/containers/agent-debian-testing-buildkite/Dockerfile +++ /dev/null @@ -1,61 +0,0 @@ -FROM debian:testing - -RUN echo "deb [trusted=yes] http://apt.llvm.org/buster/ llvm-toolchain-buster-10 main\n$(cat /etc/apt/sources.list)" > /etc/apt/sources.list ;\ - apt-get update ;\ - apt-get install -y --no-install-recommends locales \ - cmake ninja-build git ca-certificates clang lld ccache python3 build-essential gdb \ - clang-tidy clang-format \ - python3-psutil zip wget \ - openjdk-11-jdk \ - python3-pip python3-setuptools \ - swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq; \ - apt-get clean - -# Make python3 default (needed by git-clang-format and others). -RUN rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python -RUN ls /usr/bin -# required for openssh server -RUN mkdir -p /run/sshd - -ARG user=buildkite -ARG group=buildkite -ARG uid=1000 -ARG gid=1000 -ARG AGENT_WORKDIR=/home/${user}/agent - -RUN mkdir -p /scripts - -COPY start_agent.sh /scripts/ - -# install python dependencies for the scripts -# ADD will checks that contentent of a file has changed. -ADD "https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/requirements.txt" requirements.txt -RUN pip3 install wheel ; \ - pip3 install -r requirements.txt - -RUN groupadd -g ${gid} ${group} ;\ - useradd -c "buildkite user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} ;\ - mkdir /home/${user}/ccache - -RUN chown -R ${user}:${user} /home/${user} - -WORKDIR /home/${user} -ENV CCACHE_DIR=/mnt/disks/ssd0/ccache -ENV CCACHE_MAXSIZE=20G - -# configure locale -RUN sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\ - locale-gen -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 - -# buildkite -RUN apt-get install -y apt-transport-https gnupg;\ - sh -c 'echo deb https://apt.buildkite.com/buildkite-agent stable main > /etc/apt/sources.list.d/buildkite-agent.list' ;\ - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198 ;\ - apt-get update ;\ - apt-get install -y buildkite-agent - -CMD ["/scripts/start_agent.sh"] - diff --git a/containers/agent-debian-testing-buildkite/start_agent.sh b/containers/agent-debian-testing-buildkite/start_agent.sh deleted file mode 100755 index 4be51f5..0000000 --- a/containers/agent-debian-testing-buildkite/start_agent.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SSD_ROOT="/mnt/disks/ssd0" -AGENT_ROOT="${SSD_ROOT}/agent" -USER=buildkite-agent - -# prepare root folder for Jenkins agent -mkdir -p "${AGENT_ROOT}" -chown -R ${USER}:${USER} "${AGENT_ROOT}" - -# prepare folder for ccache -mkdir -p "${CCACHE_DIR}" -chown -R ${USER}:${USER} "${CCACHE_DIR}" - -# TODO(kuhnel): wipe the disk(s) on startup - -# start the buildkite agent -su buildkite-agent -c "buildkite-agent start --build-path=/mnt/disks/ssd0/agent" \ No newline at end of file diff --git a/containers/agent-debian-testing-ssd/Dockerfile b/containers/agent-debian-testing-ssd/Dockerfile deleted file mode 100644 index cc87a1a..0000000 --- a/containers/agent-debian-testing-ssd/Dockerfile +++ /dev/null @@ -1,60 +0,0 @@ -FROM debian:testing - -RUN echo "deb [trusted=yes] http://apt.llvm.org/buster/ llvm-toolchain-buster-10 main\n$(cat /etc/apt/sources.list)" > /etc/apt/sources.list;\ - apt-get update ;\ - apt-get install -y --no-install-recommends locales \ - cmake ninja-build git ca-certificates clang lld ccache python3 build-essential openssh-client\ - clang-tidy clang-format gdb \ - python3-psutil arcanist zip wget \ - openjdk-11-jdk \ - python3-pip python3-setuptools \ - swig python3-dev libedit-dev libncurses5-dev libxml2-dev liblzma-dev golang rsync jq; \ - apt-get clean - -# Make python3 default (needed by git-clang-format and others). -RUN rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python - -ARG user=jenkins -ARG group=jenkins -ARG uid=1000 -ARG gid=1000 -ARG AGENT_WORKDIR=/home/${user}/agent - -RUN mkdir -p /scripts - -# install the swarm client -RUN cd /scripts ;\ - wget https://repo.jenkins-ci.org/releases/org/jenkins-ci/plugins/swarm-client/3.17/swarm-client-3.17.jar ;\ - mv swarm-client-3.17.jar swarm-client.jar - -COPY start_agent.sh report_results.sh /scripts/ - -# store SSH known hosts for github, required for ssh authentication -RUN mkdir -p /home/${user}/.ssh -COPY known_hosts /home/${user}/.ssh/known_hosts - -# install python dependencies for the scripts -# ADD will checks that contentent of a file has changed. -ADD "https://raw.githubusercontent.com/google/llvm-premerge-checks/master/scripts/requirements.txt" requirements.txt -RUN pip3 install wheel ; \ - pip3 install -r requirements.txt - -RUN groupadd -g ${gid} ${group} ;\ - useradd -c "Jenkins user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} ;\ - mkdir /home/${user}/ccache - -RUN chown -R ${user}:${user} /home/${user} - -WORKDIR /home/${user} -ENV CCACHE_DIR=/mnt/disks/ssd0/ccache -ENV CCACHE_MAXSIZE=20G - -# configure locale -RUN sed --in-place '/en_US.UTF-8/s/^#//' /etc/locale.gen ;\ - locale-gen -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 - -CMD ["/scripts/start_agent.sh"] - diff --git a/containers/agent-debian-testing-ssd/known_hosts b/containers/agent-debian-testing-ssd/known_hosts deleted file mode 100644 index 201cf38..0000000 --- a/containers/agent-debian-testing-ssd/known_hosts +++ /dev/null @@ -1,4 +0,0 @@ -|1|bJzGXTLCQ4FZRpq+RQu+NfQOugI=|45lSmEMlpfJx7897p2Th4tZj6rM= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==|1|CA5hL0xfZtRH24/h4PieLzQaV5E=|gEuFUpdJK9mwpp1PH8RFi3DFLis= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== -|1|K6qo2Wrdv5gQipncPel2cFaNT/w=|k+coolWLGXsJ/oM4G9PBY3GLJQQ= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== -|1|hIGbHg7+Z8TQrZ/OEiRxa7f9TZs=|h6iCbIE5wV5wjMo4auBXVXgNWtU= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== -|1|PdWOrYv48xcuktJiKm97UQTg2d0=|zZImMkWTMV8HfZAUv34OvQvKyds= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== \ No newline at end of file diff --git a/containers/agent-debian-testing-ssd/report_results.sh b/containers/agent-debian-testing-ssd/report_results.sh deleted file mode 100755 index 137853d..0000000 --- a/containers/agent-debian-testing-ssd/report_results.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eux - -SUCCESS="$1" -DIFF_ID="$2" -BUILD_ID="$3" -URL="https://reviews.llvm.org" - -if [ "${SUCCESS}" == "0" ] -then - MSG="SUCCESSFUL" -else - MSG="FAILED" -fi - -arc --conduit-uri=${URL} call-conduit differential.revision.edit </dev/null 2>&1 && pwd )" diff --git a/containers/jenkins-master/Dockerfile b/containers/jenkins-master/Dockerfile deleted file mode 100644 index 63a0401..0000000 --- a/containers/jenkins-master/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM jenkins/jenkins:lts - -# disable CSRF protection, otherwise Phabricator can't trigger builds -ENV JENKINS_OPTS "-D hudson.security.csrf.GlobalCrumbIssuerConfiguration.DISABLE_CSRF_PROTECTION=true" \ No newline at end of file diff --git a/containers/nginx-results/Dockerfile b/containers/nginx-results/Dockerfile deleted file mode 100644 index 50027f9..0000000 --- a/containers/nginx-results/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM nginx:1.17 - -RUN set -e ;\ - apt-get update ;\ - apt-get install -y --no-install-recommends python3 cron;\ - apt-get clean - -RUN mkdir -p /scripts -COPY run_nginx.sh clean_results.py /scripts/ -COPY default.conf /etc/nginx/conf.d/ - -COPY crontab /etc/cron.d/clean_results -RUN chmod 0644 /etc/cron.d/clean_results -RUN service cron start - -CMD ["/scripts/run_nginx.sh"] \ No newline at end of file diff --git a/containers/nginx-results/clean_results.py b/containers/nginx-results/clean_results.py deleted file mode 100755 index f02e773..0000000 --- a/containers/nginx-results/clean_results.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/python3 -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import os -import shutil - -ROOT_DIR = '/mnt/nfs/results' -MAX_AGE = datetime.timedelta(days=90) -MAX_AGE_BIN = datetime.timedelta(days=3) - -now = datetime.datetime.now() - -for folder in [f for f in os.listdir(ROOT_DIR)]: - fullpath = os.path.join(ROOT_DIR, folder) - if not os.path.isdir(fullpath): - continue - print(fullpath) - binpath = os.path.join(ROOT_DIR, folder, 'binaries') - stats=os.stat('/tmp') - created = datetime.datetime.fromtimestamp(stats.st_mtime) - print(created) - if created + MAX_AGE < now: - print("Deleting all results: {}".format(fullpath)) - shutil.rmtree(fullpath) - elif os.path.exists(binpath) and created + MAX_AGE_BIN < now: - print("Deleting binaries: {}".format(binpath)) - shutil.rmtree(binpath) diff --git a/containers/nginx-results/crontab b/containers/nginx-results/crontab deleted file mode 100644 index f2fccc9..0000000 --- a/containers/nginx-results/crontab +++ /dev/null @@ -1,2 +0,0 @@ -* * * * * /scripty/clean_results.py -# Mandatory blank line \ No newline at end of file diff --git a/containers/nginx-results/default.conf b/containers/nginx-results/default.conf deleted file mode 100644 index 28e7368..0000000 --- a/containers/nginx-results/default.conf +++ /dev/null @@ -1,38 +0,0 @@ -server { - listen 80; - server_name results.llvm-merge-guard.org; - #charset koi8-r; - #access_log /var/log/nginx/host.access.log main; - location / { - root /mnt/nfs/results; - index index.html index.htm; - autoindex on; - } - #error_page 404 /404.html; - # redirect server error pages to the static page /50x.html - # - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } - # proxy the PHP scripts to Apache listening on 127.0.0.1:80 - # - #location ~ \.php$ { - # proxy_pass http://127.0.0.1; - #} - # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000 - # - #location ~ \.php$ { - # root html; - # fastcgi_pass 127.0.0.1:9000; - # fastcgi_index index.php; - # fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name; - # include fastcgi_params; - #} - # deny access to .htaccess files, if Apache's document root - # concurs with nginx's one - # - #location ~ /\.ht { - # deny all; - #} -} \ No newline at end of file diff --git a/containers/nginx-results/index.html b/containers/nginx-results/index.html deleted file mode 100644 index 8a21cf8..0000000 --- a/containers/nginx-results/index.html +++ /dev/null @@ -1,3 +0,0 @@ - - Hello results! - \ No newline at end of file diff --git a/containers/nginx-results/run_nginx.sh b/containers/nginx-results/run_nginx.sh deleted file mode 100755 index d1e9e8e..0000000 --- a/containers/nginx-results/run_nginx.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eux - -mkdir -p /mnt/nfs/results -chmod 777 /mnt/nfs/results -nginx -g "daemon off;" \ No newline at end of file diff --git a/docs/buildkite.md b/docs/decisions/1_migrate_from_jenkins_to_buildkite.md similarity index 100% rename from docs/buildkite.md rename to docs/decisions/1_migrate_from_jenkins_to_buildkite.md diff --git a/docs/development.md b/docs/development.md index 142a8d3..2f2faef 100644 --- a/docs/development.md +++ b/docs/development.md @@ -99,6 +99,8 @@ proxy](https://kubernetes.github.io/ingress-nginx/deploy/#gce-gke) and [certificate manager](http://docs.cert-manager.io/en/latest/getting-started/install/kubernetes.html). +[lets-encrypt configuration](../kubernetes/cert-issuer.yaml). + Access to the service is restricted with basic HTTP auth. It's configured with k8s secret 'http-auth' in 'buildkite' namespace (see [how to update auth](playbooks.md#update-http-auth-credentials)) @@ -177,4 +179,7 @@ chmod +x buildkite-agent-metrics-linux-amd64 nohup ./buildkite-agent-metrics-linux-amd64 -token XXXX -interval 30s -backend statsd & ``` -TODO: update "Testing scripts locally" playbook on how to run Linux build locally with Docker. \ No newline at end of file +Metrics are exported as "custom/statsd/gauge". + +TODO: update "Testing scripts locally" playbook on how to run Linux build locally with Docker. +TODO: migrate 'builkite-monitoring' to k8s deployment. \ No newline at end of file diff --git a/docs/playbooks.md b/docs/playbooks.md index a572a19..663a904 100644 --- a/docs/playbooks.md +++ b/docs/playbooks.md @@ -1,24 +1,22 @@ - [Playbooks](#playbooks) * [deployment to a clean infrastructure](#deployment-to-a-clean-infrastructure) - * [creating basic authentication for reverse proxy](#creating-basic-authentication-for-reverse-proxy) * [Creating docker containers on Windows](#creating-docker-containers-on-windows) * [Spawning a new windows agent](#spawning-a-new-windows-agent) + [Buildkite](#buildkite) - + [Jenkins](#jenkins) * [Testing scripts locally](#testing-scripts-locally) * [Custom environment variables](#custom-environment-variables) * [Testing changes before merging](#testing-changes-before-merging) -- [Update HTTP auth credentials](#update-http-auth-credentials) -- [Phabricator integration](#phabricator-integration) - * [Herald](#herald) - * [Harbormaster](#harbormaster) + * [Update HTTP auth credentials](#update-http-auth-credentials) # Playbooks ## deployment to a clean infrastructure General remarks: -* GCP does not route any traffic to your services unless the service is "healthy". It might take a few minutes after startup before the services is classified as healthy. Until then you will only see some generic error message. +* GCP does not route any traffic to your services unless the service is +"healthy". It might take a few minutes after startup before the services is +classified as healthy. Until then, you will only see some generic error +message. These are the steps to set up the build server on a clean infrastructure: 1. Configure the tools on your local machine: @@ -38,18 +36,6 @@ These are the steps to set up the build server on a clean infrastructure: cd kubernetes/cluster ./cluster_create.sh ``` -1. Create the disk storage, if it does not yet exist: - ```bash - cd kubernetes/cluster - ./disk_create.sh - ``` -1. SSH into the VM instance mounting the volume, find the mount point and then set - ```bash - # go to the mount point of the volume - cd /var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/jenkins-home - # change the permissions - sudo chmod a+rwx - ``` 1. Push the docker images to gcr.io: ```bash cd containers @@ -63,17 +49,6 @@ These are the steps to set up the build server on a clean infrastructure: ``` 1. Configure it -## creating basic authentication for reverse proxy - -1. create auth file, based on [ingress-nginx documentation](https://github.com/kubernetes/ingress-nginx/tree/master/docs/examples/auth/basic) - ```bash - cd kubernetes/reverse-proxy - htpasswd -c auth - # enter password at prompt - # add more users as required - kubectl create secret generic proxy-auth --from-file=auth --namespace=jenkins - ``` - ## Creating docker containers on Windows If you want to build/update/test docker container for Windows, you need to do this on a Windows machine. @@ -116,8 +91,8 @@ Here are the instructions to set up such a machine on GCP. cd c:\ git clone https://github.com/google/llvm-premerge-checks cd llvm-premerge-checks\containers - .\build_deploy.ps1 agent-windows-buildkite # or agent-windows-jenkins - c:\llvm-premerge-check\scripts\windows_agent_start_buildkite.ps1 # or windows_agent_start_jenkins.ps1 + .\build_deploy.ps1 agent-windows-buildkite + c:\llvm-premerge-check\scripts\windows_agent_start_buildkite.ps1 ``` ## Spawning a new windows agent @@ -155,22 +130,10 @@ To spawn a new windows agent: git clone https://github.com/google/llvm-premerge-checks.git C:\llvm-premerge-checks schtasks.exe /create /tn "Start Buildkite agent" /ru SYSTEM /SC ONSTART /DELAY 0005:00 /tr "powershell -command 'C:\llvm-premerge-checks\scripts\windows_agent_start_buildkite.ps1 -workdir c:\ws'" ``` - -### Jenkins - 1. Create `c:\credentials` folder with `build-agent-results_key.json` to access cloud storage copy from one of the existing machines. - 1. Run - ```powershell - git clone https://github.com/google/llvm-premerge-checks.git "c:\llvm-premerge-checks" - C:\llvm-premerge-checks\scripts\windows_agent_start_buildkite.ps1 [-testing] [-version latest] - ``` - -Metrics are exported as "custom/statsd/gauge". ## Testing scripts locally -Build and run agent docker image `sudo ./containers/build_run.sh agent-debian-testing-ssd /bin/bash`. - -Within a container set environment variables similar to [pipeline](https://github.com/google/llvm-premerge-checks/blob/master/Jenkins/Phabricator-pipeline/Jenkinsfile). +Build and run agent docker image `sudo ./containers/build_run.sh buildkite-premerge-debian /bin/bash`. Additionally set `WORKSPACE`, `PHID` and `DIFF_ID` parameters. Set `CONDUIT_TOKEN` with your personal one from `https://reviews.llvm.org/settings/user//page/apitokens/`. @@ -200,7 +163,7 @@ It's recommended to test even smallest changes before committing them to the `ma `scripts_branch="my-feature"` (see also "Custom environment variables" for other options above). To test "premerge-tests" pipeline pick an existing build and copy parameters from it, omitting "ph_target_phid", namely: "ph_build_id", "ph_buildable_diff", "ph_buildable_revision", "ph_initiator_phid" and "scripts_branch" variables. 1. Wait for build to complete and maybe attach a link to it to your PR. -# Update HTTP auth credentials +## Update HTTP auth credentials To update e.g. buildkite http-auth: ```shell script @@ -212,32 +175,4 @@ htpasswd -b auth # update secret kubectl delete secret http-auth -n buildkite kubectl create secret generic http-auth -n buildkite --from-file=./auth -``` - -# Phabricator integration - -The general flow for builds on Phabricator is: -1. A user uploads a *Diff* (=patch) to a *Revision* (set of Diffs with comments and buildstatus, ... ). -2. A *Herald* checks if one of the *rules* matches this event. -3. You can use the rules to trigger a *Build* in *Harbormaster*. -4. Harbor sends an HTTP request to the Jenkins server. -5. Jenkins executes the build. In the last step of the build, a script is uploading the results to Phabricator. -6. Phabricator sets the build status and displays the results. - -## Herald - -We currently have these Herald rules to configure the builds: -* Triggering builds for everyone: - * [H576](https://reviews.llvm.org/H576) This will only trigger for non-beta testers. -* Triggering the beta-test builds: - * [H511](https://reviews.llvm.org/H511) or the beta testers, this is for testing new features. - * [H552](https://reviews.llvm.org/H552) for all changes to MLIR (archived) - * [H527](https://reviews.llvm.org/H527) for all changes to clang-extra-tools (archived) - -You can *archive* a rule to disable it. - -## Harbormaster - -We have these build plans in Harbormaster: -* [Plan 4](https://reviews.llvm.org/harbormaster/plan/4/) Builds for everyone -* [Plan 3](https://reviews.llvm.org/harbormaster/plan/3/) Builds for beta testers +``` \ No newline at end of file diff --git a/kubernetes/buildkite/kustomization.yaml b/kubernetes/buildkite/kustomization.yaml deleted file mode 100644 index 4cd9a42..0000000 --- a/kubernetes/buildkite/kustomization.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: jenkins -resources: - - deployment.yaml - diff --git a/kubernetes/buildkite/deployment.yaml b/kubernetes/buildkite/linux-agents.yaml similarity index 89% rename from kubernetes/buildkite/deployment.yaml rename to kubernetes/buildkite/linux-agents.yaml index 036961c..a6624ad 100644 --- a/kubernetes/buildkite/deployment.yaml +++ b/kubernetes/buildkite/linux-agents.yaml @@ -15,7 +15,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: premerge-debian + name: linux-agents namespace: buildkite spec: replicas: 4 @@ -53,8 +53,12 @@ spec: secretKeyRef: name: buildkite-agent-token key: token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name - name: BUILDKITE_AGENT_TAGS - value: "queue=linux" + value: "queue=linux,name=$(POD_NAME)" - name: BUILDKITE_BUILD_PATH value: "/mnt/disks/ssd0/agent" - name: CONDUIT_TOKEN @@ -72,4 +76,4 @@ spec: secret: secretName: github-ssh nodeSelector: - cloud.google.com/gke-nodepool: jenkins-agents \ No newline at end of file + cloud.google.com/gke-nodepool: linux-agents \ No newline at end of file diff --git a/kubernetes/reverse-proxy/Issuer.yaml b/kubernetes/cert-issuer.yaml similarity index 100% rename from kubernetes/reverse-proxy/Issuer.yaml rename to kubernetes/cert-issuer.yaml diff --git a/kubernetes/cluster_create.sh b/kubernetes/cluster_create.sh index ec46d1d..0bb84f6 100755 --- a/kubernetes/cluster_create.sh +++ b/kubernetes/cluster_create.sh @@ -25,10 +25,9 @@ source "${ROOT_DIR}/k8s_config" gcloud container clusters create $GCP_CLUSTER --zone $GCP_ZONE \ --machine-type=n1-standard-4 --num-nodes=1 -# Jenkins agents with local ssd -# as per instructions +# Linux agents node pool with local ssd. # https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/local-ssd -gcloud container node-pools create jenkins-agents --cluster $GCP_CLUSTER --zone $GCP_ZONE \ +gcloud container node-pools create linux-agents --cluster $GCP_CLUSTER --zone $GCP_ZONE \ --machine-type=n1-standard-32 --num-nodes=2 --local-ssd-count=1 # created separate cluster for windows, as we need "ip-alias" enabled diff --git a/kubernetes/deploy.sh b/kubernetes/deploy.sh deleted file mode 100755 index 85d01e0..0000000 --- a/kubernetes/deploy.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eux - -# nfs needs to be deployed first, as it creates the presistent volumes -kubectl apply -f nfs.yaml -kubectl apply -k results-nginx -kubectl apply -f jenkins.yaml \ No newline at end of file diff --git a/kubernetes/disk_create.sh b/kubernetes/disk_create.sh deleted file mode 100755 index 197d402..0000000 --- a/kubernetes/disk_create.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eux - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -ROOT_DIR="$(dirname ${DIR})" - -# get config options -source "${ROOT_DIR}/k8s_config" - -gcloud compute disks create jenkins-home \ - --description="storage for jenkins master" \ - --size=200GB \ - --type=pd-standard \ - --zone=${GCP_ZONE} \ - -gcloud compute disks create jenkins-home-staging \ - --description="storage for jenkins master (staging)" \ - --size=20GB \ - --type=pd-standard \ - --zone=${GCP_ZONE} \ - -gcloud compute disks create results \ - --description="storage build results" \ - --size=20GB \ - --type=pd-standard \ - --zone=${GCP_ZONE} diff --git a/kubernetes/disk_delete.sh b/kubernetes/disk_delete.sh deleted file mode 100755 index 1e9ca37..0000000 --- a/kubernetes/disk_delete.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eux - -# to detatch: -# gcloud compute instances detach-disk --disk=jenkins-home -gcloud compute disks delete jenkins-home \ No newline at end of file diff --git a/kubernetes/jenkins-staging/Deployment.yaml b/kubernetes/jenkins-staging/Deployment.yaml deleted file mode 100644 index 27b27b3..0000000 --- a/kubernetes/jenkins-staging/Deployment.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: jenkins-staging - namespace: jenkins -spec: - replicas: 1 - template: - metadata: - labels: - app: master-staging - spec: - hostname: jenkins-master-staging - containers: - - name: master-staging - image: gcr.io/llvm-windows-development/jenkins-master - ports: - - containerPort: 8080 - - containerPort: 50000 - readinessProbe: - httpGet: - path: /login - port: 8080 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 2 - failureThreshold: 5 - env: - - name: JAVA_OPTS - value: '-Xmx1400m' - volumeMounts: - - mountPath: /var/jenkins_home - name: jenkins-home-staging - resources: - limits: - cpu: 500m - memory: 1500Mi - requests: - cpu: 500m - memory: 1500Mi - volumes: - - name: jenkins-home-staging - gcePersistentDisk: - pdName: jenkins-home-staging - fsType: ext4 - nodeSelector: - cloud.google.com/gke-nodepool: default-pool ---- diff --git a/kubernetes/jenkins-staging/PersistentVolume.yaml b/kubernetes/jenkins-staging/PersistentVolume.yaml deleted file mode 100644 index b6fc97e..0000000 --- a/kubernetes/jenkins-staging/PersistentVolume.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: PersistentVolume -metadata: - name: jenkins-home-staging - labels: - failure-domain.beta.kubernetes.io/zone: us-central1-a -spec: - capacity: - storage: 20Gi - volumeMode: Filesystem - accessModes: - - ReadWriteOnce - gcePersistentDisk: - pdName: jenkins-home-staging - fsType: ext4 \ No newline at end of file diff --git a/kubernetes/jenkins-staging/Services.yaml b/kubernetes/jenkins-staging/Services.yaml deleted file mode 100644 index 8cb587c..0000000 --- a/kubernetes/jenkins-staging/Services.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -kind: Service -apiVersion: v1 -metadata: - name: jenkins-ui-staging - namespace: jenkins -spec: - type: NodePort - selector: - app: master-staging - ports: - - protocol: TCP - port: 8080 - targetPort: 8080 - name: ui ---- diff --git a/kubernetes/jenkins-staging/kustomization.yaml b/kubernetes/jenkins-staging/kustomization.yaml deleted file mode 100644 index d8fb812..0000000 --- a/kubernetes/jenkins-staging/kustomization.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: jenkins -resources: - - PersistentVolume.yaml - - Deployment.yaml - - Services.yaml - diff --git a/kubernetes/jenkins.yaml b/kubernetes/jenkins.yaml deleted file mode 100644 index 4e54676..0000000 --- a/kubernetes/jenkins.yaml +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -apiVersion: v1 -kind: Namespace -metadata: - name: jenkins - labels: - name: jenkins ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: default - namespace: jenkins ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: jenkins-admin -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: - - kind: ServiceAccount - name: default - namespace: jenkins ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: jenkins-home - labels: - failure-domain.beta.kubernetes.io/zone: us-central1-a -spec: - capacity: - storage: 20Gi - volumeMode: Filesystem - accessModes: - - ReadWriteOnce - gcePersistentDisk: - pdName: jenkins-home - fsType: ext4 ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: jenkins - namespace: jenkins -spec: - replicas: 1 - template: - metadata: - labels: - app: master - spec: - hostname: jenkins-master - containers: - - name: master - image: jenkins/jenkins:2.190.3 - ports: - - containerPort: 8080 - - containerPort: 50000 - readinessProbe: - httpGet: - path: /login - port: 8080 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 2 - failureThreshold: 5 - env: - - name: JAVA_OPTS - value: '-Xmx1400m' - volumeMounts: - - mountPath: /var/jenkins_home - name: jenkins-home - resources: - limits: - cpu: "4" - memory: 4000Mi - requests: - cpu: 500m - memory: 2000Mi - volumes: - - name: jenkins-home - gcePersistentDisk: - pdName: jenkins-home - fsType: ext4 - nodeSelector: - cloud.google.com/gke-nodepool: default-pool ---- -kind: Service -apiVersion: v1 -metadata: - name: jenkins-ui - namespace: jenkins -spec: - type: NodePort - selector: - app: master - ports: - - protocol: TCP - port: 8080 - targetPort: 8080 - name: ui - - protocol: TCP - port: 50000 - targetPort: 50000 - name: agent ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: nfs-jenkins -spec: - capacity: - storage: 10Gi - accessModes: - - ReadWriteMany - nfs: - server: nfs-service.results.svc.cluster.local - path: "/exports" ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: nfs-jenkins - namespace: jenkins -spec: - accessModes: - - ReadWriteMany - storageClassName: "" - resources: - requests: - storage: 10Gi ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: agent-debian-testing-ssd - namespace: jenkins -spec: - replicas: 2 - template: - metadata: - labels: - app: agent-debian-testing-ssd - spec: - containers: - - name: agent-debian-testing-clang8 - image: gcr.io/llvm-premerge-checks/agent-debian-testing-ssd - ports: - - containerPort: 22 - resources: - limits: - cpu: 30 - memory: 45Gi - requests: - cpu: 30 - memory: 45Gi - volumeMounts: - - name: nfs-pvc - mountPath: /mnt/nfs - - name: ssd - mountPath: /mnt/disks/ssd0 - - mountPath: /github-ssh-key - name: github-ssh-key - volumes: - - name: nfs-pvc - persistentVolumeClaim: - claimName: nfs-jenkins - - name: ssd - hostPath: - # directory location on host - path: /mnt/disks/ssd0 - type: Directory - - name: github-ssh-key - secret: - secretName: github-ssh-key - nodeSelector: - cloud.google.com/gke-nodepool: jenkins-agents diff --git a/kubernetes/nfs.yaml b/kubernetes/nfs.yaml deleted file mode 100644 index 8aa3274..0000000 --- a/kubernetes/nfs.yaml +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -apiVersion: v1 -kind: Namespace -metadata: - name: results - labels: - name: results ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: nfs-server - namespace: results -spec: - replicas: 1 - selector: - matchLabels: - role: nfs-server - template: - metadata: - labels: - role: nfs-server - spec: - containers: - - name: nfs-server - image: gcr.io/google_containers/volume-nfs - ports: - - name: nfs - containerPort: 2049 - - name: mountd - containerPort: 20048 - - name: rpcbind - containerPort: 111 - securityContext: - privileged: true - volumeMounts: - - mountPath: /exports - name: result-pvc - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 500m - memory: 500Mi - volumes: - - name: result-pvc - gcePersistentDisk: - pdName: results-new - fsType: ext4 - nodeSelector: - cloud.google.com/gke-nodepool: default-pool ---- -apiVersion: v1 -kind: Service -metadata: - name: nfs-service - namespace: results -spec: - ports: - - name: nfs - port: 2049 - - name: mountd - port: 20048 - - name: rpcbind - port: 111 - selector: - role: nfs-server ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: nfs -spec: - capacity: - storage: 20Gi - accessModes: - - ReadWriteMany - nfs: - server: nfs-service.results.svc.cluster.local - path: "/exports" ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: nfs - namespace: results -spec: - accessModes: - - ReadWriteMany - storageClassName: "" - resources: - requests: - storage: 1Gi \ No newline at end of file diff --git a/kubernetes/phabricator-proxy/kustomization.yaml b/kubernetes/phabricator-proxy/kustomization.yaml index 66bec82..828e589 100644 --- a/kubernetes/phabricator-proxy/kustomization.yaml +++ b/kubernetes/phabricator-proxy/kustomization.yaml @@ -14,7 +14,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -namespace: jenkins +namespace: buildkite resources: - Deployment.yaml - Services.yaml diff --git a/kubernetes/results-nginx/Deployment.yaml b/kubernetes/results-nginx/Deployment.yaml deleted file mode 100644 index 89acc67..0000000 --- a/kubernetes/results-nginx/Deployment.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-results - namespace: jenkins -spec: - selector: - matchLabels: - app: nginx-results - replicas: 1 - template: - metadata: - labels: - app: nginx-results - spec: - containers: - - name: nginx-results - image: gcr.io/llvm-premerge-checks/nginx-results - ports: - - containerPort: 80 - readinessProbe: - httpGet: - path: / - port: 80 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 2 - failureThreshold: 5 - volumeMounts: - - mountPath: /mnt/nfs - name: nfs-pvc - resources: - limits: - cpu: 500m - memory: 1500Mi - requests: - cpu: 500m - memory: 1500Mi - volumes: - - name: nfs-pvc - persistentVolumeClaim: - claimName: nfs-jenkins - nodeSelector: - cloud.google.com/gke-nodepool: default-pool \ No newline at end of file diff --git a/kubernetes/results-nginx/Services.yaml b/kubernetes/results-nginx/Services.yaml deleted file mode 100644 index 929111d..0000000 --- a/kubernetes/results-nginx/Services.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kind: Service -apiVersion: v1 -metadata: - name: nginx-results - namespace: jenkins -spec: -# type: LoadBalancer -# externalTrafficPolicy: Cluster - selector: - app: nginx-results - ports: - - protocol: TCP - port: 80 - targetPort: 80 diff --git a/kubernetes/results-nginx/kustomization.yaml b/kubernetes/results-nginx/kustomization.yaml deleted file mode 100644 index 2071079..0000000 --- a/kubernetes/results-nginx/kustomization.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: jenkins -resources: - - Deployment.yaml - - Services.yaml - diff --git a/kubernetes/reverse-proxy/.gitignore b/kubernetes/reverse-proxy/.gitignore deleted file mode 100644 index 9ec0d09..0000000 --- a/kubernetes/reverse-proxy/.gitignore +++ /dev/null @@ -1 +0,0 @@ -auth \ No newline at end of file diff --git a/kubernetes/reverse-proxy/Ingress.yaml b/kubernetes/reverse-proxy/Ingress.yaml deleted file mode 100644 index 40739dd..0000000 --- a/kubernetes/reverse-proxy/Ingress.yaml +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: nginx-ingress-prod-jenkins - namespace: jenkins - annotations: - # static IP assignment not working. Not sure why. - kubernetes.io/ingress.global-static-ip-name: "web-static-ip" - kubernetes.io/ingress.class: "nginx" - cert-manager.io/cluster-issuer: "letsencrypt-prod" - nginx.ingress.kubernetes.io/auth-type: basic - nginx.ingress.kubernetes.io/auth-secret: proxy-auth - nginx.ingress.kubernetes.io/auth-realm: "Authentication Required - LLVM pre-merge checks" - nginx.ingress.kubernetes.io/configuration-snippet: | - proxy_set_header Authorization $remote_user; -spec: - tls: - - secretName: jenkins-prod-tls - hosts: - - jenkins.llvm-merge-guard.org - - rules: - - host: jenkins.llvm-merge-guard.org - http: - paths: - - backend: - serviceName: jenkins-ui - servicePort: 8080 - ---- -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: nginx-ingress-prod-results - namespace: jenkins - annotations: - # static IP assignment not working. Not sure why. - kubernetes.io/ingress.global-static-ip-name: "web-static-ip" - kubernetes.io/ingress.class: "nginx" - cert-manager.io/cluster-issuer: "letsencrypt-prod" -spec: - tls: - - secretName: results-prod-tls - hosts: - - results.llvm-merge-guard.org - rules: - - host: results.llvm-merge-guard.org - http: - paths: - - backend: - serviceName: nginx-results - servicePort: 80 -# --- -# apiVersion: extensions/v1beta1 -# kind: Ingress -# metadata: -# name: nginx-ingress-staging-jenkins -# namespace: jenkins -# annotations: -# # static IP assignment not working. Not sure why. -# kubernetes.io/ingress.global-static-ip-name: "web-static-ip" -# kubernetes.io/ingress.class: "nginx" -# cert-manager.io/cluster-issuer: "letsencrypt-staging" -# nginx.ingress.kubernetes.io/auth-type: basic -# nginx.ingress.kubernetes.io/auth-secret: proxy-auth -# nginx.ingress.kubernetes.io/auth-realm: "Authentication Required - LLVM pre-merge checks" -# nginx.ingress.kubernetes.io/configuration-snippet: | -# proxy_set_header Authorization $remote_user; -# spec: -# tls: -# - secretName: jenkins-staging-tls -# hosts: -# - jenkins.staging.llvm-merge-guard.org - -# rules: -# - host: jenkins.staging.llvm-merge-guard.org -# http: -# paths: -# - backend: -# serviceName: jenkins-ui-staging -# servicePort: 8080 -# --- - -# apiVersion: extensions/v1beta1 -# kind: Ingress -# metadata: -# name: nginx-ingress-staging-results -# namespace: jenkins -# annotations: -# # static IP assignment not working. Not sure why. -# kubernetes.io/ingress.global-static-ip-name: "web-static-ip" -# kubernetes.io/ingress.class: "nginx" -# cert-manager.io/cluster-issuer: "letsencrypt-staging" -# spec: -# tls: -# - secretName: results-staging-tls -# hosts: -# - results.staging.llvm-merge-guard.org -# rules: -# - host: results.staging.llvm-merge-guard.org -# http: -# paths: -# - backend: -# serviceName: nginx-results -# servicePort: 80 -# --- \ No newline at end of file diff --git a/kubernetes/reverse-proxy/Namespace.yaml b/kubernetes/reverse-proxy/Namespace.yaml deleted file mode 100644 index 3c554ea..0000000 --- a/kubernetes/reverse-proxy/Namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: jenkins - labels: - name: jenkins \ No newline at end of file diff --git a/kubernetes/reverse-proxy/add_user.sh b/kubernetes/reverse-proxy/add_user.sh deleted file mode 100755 index 8b86610..0000000 --- a/kubernetes/reverse-proxy/add_user.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -eux - -USER_NAME="${1}" -AUTH_FILE="${HOME}/.llvm-premerge-checks/auth" - -if [ -f "${AUTH_FILE}" ] ; then - htpasswd "${AUTH_FILE}" "${USER_NAME}" -else - mkdir -p "$(dirname "${AUTH_FILE}")" - htpasswd -c "${AUTH_FILE}" "${USER_NAME}" -fi \ No newline at end of file diff --git a/kubernetes/reverse-proxy/basic.sh b/kubernetes/reverse-proxy/basic.sh deleted file mode 100755 index a82306c..0000000 --- a/kubernetes/reverse-proxy/basic.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eux - -# basic setup of the reverse proxy based on -# https://kubernetes.github.io/ingress-nginx/deploy/ -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/mandatory.yaml -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/cloud-generic.yaml - -# install certmanager based on -# http://docs.cert-manager.io/en/latest/getting-started/install/kubernetes.html - -kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.15.0/cert-manager.yaml -kubectl create clusterrolebinding cluster-admin-binding \ - --clusterrole=cluster-admin \ - --user=$(gcloud config get-value core/account) - diff --git a/kubernetes/reverse-proxy/kustomization.yaml b/kubernetes/reverse-proxy/kustomization.yaml deleted file mode 100644 index 17ea2f4..0000000 --- a/kubernetes/reverse-proxy/kustomization.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: jenkins -resources: - - Namespace.yaml - - Issuer.yaml - - Certificates.yaml - - Ingress.yaml - diff --git a/kubernetes/reverse-proxy/update_auth.sh b/kubernetes/reverse-proxy/update_auth.sh deleted file mode 100755 index 4cb84f8..0000000 --- a/kubernetes/reverse-proxy/update_auth.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -eux - -AUTH_FILE="${HOME}/.llvm-premerge-checks/auth" - -if [ ! -f "${AUTH_FILE}" ] ; then - echo "Error: Did not find auth file in: ${AUTH_FILE}!" - exit 1 -fi - -#delete the old secret -kubectl delete secret proxy-auth --namespace=jenkins -# upload the new secret -kubectl create secret generic proxy-auth --from-file="${AUTH_FILE}" --namespace=jenkins - -echo "Done." \ No newline at end of file diff --git a/kubernetes/store_ssh_key.sh b/kubernetes/store_ssh_key.sh deleted file mode 100755 index 955086a..0000000 --- a/kubernetes/store_ssh_key.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#------------------------------------------------------------------------------- -set -eux - -# This scripts creates a new ssh keypair (if it does not exist) and uploads -# it to a kubernetes secret. -# -# You need to manually upload the public key to Github so that you can use -# it for authentication. - - -LOCAL_SSH_DIR="$HOME/.llvm-premerge-checks/github-ssh" - -if [ ! -d "$LOCAL_SSH_DIR" ]; then - mkdir -p "$LOCAL_SSH_DIR" - pushd "$LOCAL_SSH_DIR" - ssh-keygen -b 4096 -t rsa -f "$LOCAL_SSH_DIR/id_rsa" -q -N "" - popd -fi - -kubectl create secret generic github-ssh-key --namespace jenkins \ - --from-file "$LOCAL_SSH_DIR/id_rsa" \ - --from-file "$LOCAL_SSH_DIR/id_rsa.pub" \ No newline at end of file diff --git a/kubernetes/windows-agents/Deployment.yaml b/kubernetes/windows-agents/Deployment.yaml deleted file mode 100644 index 4e00384..0000000 --- a/kubernetes/windows-agents/Deployment.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: agent-windows-visualstudio2017-ssd - labels: - app: agent-windows-visualstudio2017-ssd -spec: - replicas: 1 - selector: - matchLabels: - app: agent-windows-visualstudio2017-ssd - template: - metadata: - labels: - app: agent-windows-visualstudio2017-ssd - spec: - containers: - - name: agent-windows-visualstudio2017-ssd - image: gcr.io/llvm-premerge-checks/agent-windows-jenkins - resources: - limits: - cpu: 14 - memory: 20Gi - requests: - cpu: 14 - memory: 20Gi -# volumeMounts: -# - name: nfs-pvc -# mountPath: /mnt/nfs -# - name: ssd -# mountPath: /mnt/disks/ssd0 -# volumes: -# - name: nfs-pvc -# persistentVolumeClaim: -# claimName: nfs-jenkins -# - name: ssd -# hostPath: -# # directory location on host -# path: /mnt/disks/ssd0 -# type: Directory - nodeSelector: - kubernetes.io/os: windows - cloud.google.com/gke-nodepool: windows-pool \ No newline at end of file diff --git a/kubernetes/windows-agents/kustomization.yaml b/kubernetes/windows-agents/kustomization.yaml deleted file mode 100644 index 521cb26..0000000 --- a/kubernetes/windows-agents/kustomization.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - Deployment.yaml - diff --git a/kubernetes/windows_agent_create.sh b/kubernetes/windows_agent_create.sh index eca813e..09a9495 100755 --- a/kubernetes/windows_agent_create.sh +++ b/kubernetes/windows_agent_create.sh @@ -21,7 +21,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" ROOT_DIR="$(dirname ${DIR})" # get config options -# shellcheck source=../k8s_config source "${ROOT_DIR}/k8s_config" NAME=$1 diff --git a/scripts/clang-tidy-comments.ignore b/scripts/clang-tidy-comments.ignore index 6663111..b782762 100644 --- a/scripts/clang-tidy-comments.ignore +++ b/scripts/clang-tidy-comments.ignore @@ -1,4 +1,5 @@ # Files that are allowed by clang-tidy.ignore but should not receive inline review comments. # Right now it works in whitelist mode and only some files / directories are whitelisted. +# FIXME: is is used? * !clang-tools-extra/clangd/** \ No newline at end of file diff --git a/scripts/lint.sh b/scripts/lint.sh deleted file mode 100755 index 308d96c..0000000 --- a/scripts/lint.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Runs clang-format -# Inputs: TARGET_DIR, WORKSPACE -# Outputs: ${TARGET_DIR}/clang-format.patch (if there are clang-format findings). -set -eux - -echo "Running linters... =====================================" -if (( $# != 2 )); then - echo "Syntax: lint.sh " - exit 1 -fi; -# Commit to diff against -COMMIT="$1" -# output directory for test results -OUTPUT_DIR="$2" -# root directory, where the config files are located -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -if [ ! -f "compile_commands.json" ] ; then - echo "Could not find compile commands.json in $(pwd)" - exit 1 -fi - -# clang-format -# Let clang format apply patches --diff doesn't produces results in the format we want. -git-clang-format "${COMMIT}" -set +e -git diff -U0 --exit-code --no-prefix | "${DIR}/ignore_diff.py" "${DIR}/clang-format.ignore" > "${OUTPUT_DIR}"/clang-format.patch -set -e -# Revert changes of git-clang-format. -git checkout -- . - -# clang-tidy -git diff -U0 --no-prefix "${COMMIT}" | "${DIR}/ignore_diff.py" "${DIR}/clang-tidy.ignore" | clang-tidy-diff -p0 -quiet | sed "/^[[:space:]]*$/d" > "${OUTPUT_DIR}"/clang-tidy.txt - -echo "linters completed ======================================" diff --git a/scripts/phabtalk/add_url_artifact.py b/scripts/phabtalk/add_url_artifact.py index 81bcdd2..32da386 100755 --- a/scripts/phabtalk/add_url_artifact.py +++ b/scripts/phabtalk/add_url_artifact.py @@ -21,8 +21,6 @@ import uuid if __name__ == '__main__': sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -# from phabtalk import PhabTalk -# else: from phabtalk.phabtalk import PhabTalk diff --git a/scripts/phabtalk/phabtalk.py b/scripts/phabtalk/phabtalk.py index 15d0c1a..1908fc7 100755 --- a/scripts/phabtalk/phabtalk.py +++ b/scripts/phabtalk/phabtalk.py @@ -12,23 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Upload build results to Phabricator. +""" +Interactions with Phabricator. +""" -As I did not like the Jenkins plugin, we're using this script to upload the -build status, a summary and the test reults to Phabricator.""" - -import argparse -import os -import re import socket import time -import urllib import uuid from typing import Optional, List, Dict -import pathspec -from lxml import etree from phabricator import Phabricator -from enum import IntEnum class PhabTalk: @@ -150,49 +142,6 @@ class PhabTalk: artifactData=artifact_data)) -def _parse_patch(patch) -> List[Dict[str, str]]: - """Extract the changed lines from `patch` file. - The return value is a list of dictionaries {filename, line, diff}. - Diff must be generated with -U0 (no context lines). - """ - entries = [] - lines = [] - filename = None - line_number = 0 - for line in patch: - match = re.search(r'^(\+\+\+|---) [^/]+/(.*)', line) - if match: - if len(lines) > 0: - entries.append({ - 'filename': filename, - 'diff': ''.join(lines), - 'line': line_number, - }) - lines = [] - filename = match.group(2).rstrip('\r\n') - continue - match = re.search(r'^@@ -(\d+)(,(\d+))? \+(\d+)(,(\d+))?', line) - if match: - if len(lines) > 0: - entries.append({ - 'filename': filename, - 'diff': ''.join(lines), - 'line': line_number, - }) - lines = [] - line_number = int(match.group(1)) - continue - if line.startswith('+') or line.startswith('-'): - lines.append(line) - if len(lines) > 0: - entries.append({ - 'filename': filename, - 'diff': ''.join(lines), - 'line': line_number, - }) - return entries - - class Step: def __init__(self): self.name = '' @@ -236,292 +185,6 @@ class Report: self.artifacts.append({'dir': dir, 'file': file, 'name': name}) -class BuildReport: - - def __init__(self, args): - # self.args = args - self.ph_id = args.ph_id # type: str - self.diff_id = args.diff_id # type: str - self.test_result_file = args.test_result_file # type: str - self.conduit_token = args.conduit_token # type: str - self.dryrun = args.dryrun # type: bool - self.buildresult = args.buildresult # type: str - self.clang_format_patch = args.clang_format_patch # type: str - self.clang_tidy_result = args.clang_tidy_result # type: str - self.clang_tidy_ignore = args.clang_tidy_ignore # type: str - self.results_dir = args.results_dir # type: str - self.results_url = args.results_url # type: str - self.workspace = args.workspace # type: str - self.failure_messages = args.failures # type: str - self.name = args.name # type: str - - self.api = PhabTalk(args.conduit_token, args.host, args.dryrun) - - self.revision_id = self.api.get_revision_id(self.diff_id) - self.comments = [] - self.success = True - self.working = False - self.unit = [] # type: List - self.lint = {} - self.test_stats = { - 'pass': 0, - 'fail': 0, - 'skip': 0 - } # type: Dict[str, int] - - def add_lint(self, m): - key = '{}:{}'.format(m['path'], m['line']) - if key not in self.lint: - self.lint[key] = [] - self.lint[key].append(m) - - def final_report(self): - if self.buildresult is not None: - print('Jenkins result: {}'.format(self.buildresult)) - if self.buildresult.lower() == 'success': - pass - elif self.buildresult.lower() == 'null': - self.working = True - else: - self.success = False - else: - self.success = False - - try: - self.add_test_results() - except etree.XMLSyntaxError: - # Sometimes we get an incomplete XML file. - # In this case: - # - fail the build (the safe thing to do) - # - continue so the user gets some feedback. - print('Error parsing {}. Invalid XML syntax!'.format(self.test_result_file)) - self.success = False - - self.add_clang_tidy() - self.add_clang_format() - self.api.update_build_status(self.diff_id, self.ph_id, self.working, self.success, self.lint, self.unit) - - self.add_links_to_artifacts() - - title = 'Issue with build for {} ({})'.format(self.api.get_revision_id(self.diff_id), self.diff_id) - self.comments.append( - 'Pre-merge checks is in beta report issue.
' - 'Please join beta or ' - 'enable it for your project'.format( - urllib.parse.quote(title))) - with open(os.path.join(self.results_dir, 'summary.html'), 'w') as f: - f.write('') - f.write('

Build result for diff {0} {1} at {2}

'.format( - self.revision_id, self.diff_id, self.name)) - if self.failure_messages and len(self.failure_messages) > 0: - for s in self.failure_messages.split('\n'): - f.write('

{}

'.format(s)) - f.write('

' + '

'.join(self.comments) + '

') - f.write('') - self.api.add_artifact(self.ph_id, 'summary.html', 'summary ' + self.name, self.results_url) - - def add_clang_format(self): - """Populates results from diff produced by clang format.""" - if self.clang_format_patch is None: - return - present = os.path.exists( - os.path.join(self.results_dir, self.clang_format_patch)) - if not present: - print('clang-format result {} is not found'.format(self.clang_format_patch)) - self.comments.append(section_title('clang-format', False, False)) - return - p = os.path.join(self.results_dir, self.clang_format_patch) - if os.stat(p).st_size != 0: - self.api.add_artifact(self.ph_id, self.clang_format_patch, 'clang-format ' + self.name, self.results_url) - diffs = _parse_patch(open(p, 'r')) - success = len(diffs) == 0 - for d in diffs: - lines = d['diff'].splitlines(keepends=True) - m = 10 # max number of lines to report. - description = 'please reformat the code\n```\n' - n = len(lines) - cut = n > m + 1 - if cut: - lines = lines[:m] - description += ''.join(lines) + '\n```' - if cut: - description += '\n{} diff lines are omitted. See [full diff]({}/{}).'.format( - n - m, - self.results_url, - self.clang_format_patch) - self.add_lint({ - 'name': 'clang-format', - 'severity': 'autofix', - 'code': 'clang-format', - 'path': d['filename'], - 'line': d['line'], - 'char': 1, - 'description': description, - }) - comment = section_title('clang-format', success, present) - if not success: - comment += 'Please format your changes with clang-format by running `git-clang-format HEAD^` or applying ' \ - 'this patch.'.format(os.path.basename(self.clang_format_patch)) - self.comments.append(comment) - self.success = success and self.success - - def add_clang_tidy(self): - if self.clang_tidy_result is None: - return - # Typical message looks like - # [..]/clang/include/clang/AST/DeclCXX.h:3058:20: error: no member named 'LifetimeExtendedTemporary' in 'clang::Decl' [clang-diagnostic-error] - pattern = '^{}/([^:]*):(\\d+):(\\d+): (.*): (.*)'.format(self.workspace) - errors_count = 0 - warn_count = 0 - inline_comments = 0 - present = os.path.exists( - os.path.join(self.results_dir, self.clang_tidy_result)) - if not present: - print('clang-tidy result {} is not found'.format(self.clang_tidy_result)) - self.comments.append(section_title('clang-tidy', False, False)) - return - present = (self.clang_tidy_ignore is not None) and os.path.exists(self.clang_tidy_ignore) - if not present: - print('clang-tidy ignore file {} is not found'.format(self.clang_tidy_ignore)) - self.comments.append(section_title('clang-tidy', False, False)) - return - p = os.path.join(self.results_dir, self.clang_tidy_result) - add_artifact = False - ignore = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, - open(self.clang_tidy_ignore, 'r').readlines()) - for line in open(p, 'r'): - line = line.strip() - if len(line) == 0 or line == 'No relevant changes found.': - continue - add_artifact = True - match = re.search(pattern, line) - if match: - file_name = match.group(1) - line_pos = match.group(2) - char_pos = match.group(3) - severity = match.group(4) - text = match.group(5) - text += '\n[[{} | not useful]] '.format( - 'https://github.com/google/llvm-premerge-checks/blob/master/docs/clang_tidy.md#warning-is-not' - '-useful') - if severity in ['warning', 'error']: - if severity == 'warning': - warn_count += 1 - if severity == 'error': - errors_count += 1 - if ignore.match_file(file_name): - print('{} is ignored by pattern and no comment will be added'.format(file_name)) - else: - inline_comments += 1 - self.add_lint({ - 'name': 'clang-tidy', - 'severity': 'warning', - 'code': 'clang-tidy', - 'path': file_name, - 'line': int(line_pos), - 'char': int(char_pos), - 'description': '{}: {}'.format(severity, text), - }) - if add_artifact: - self.api.add_artifact(self.ph_id, self.clang_tidy_result, 'clang-tidy ' + self.name, self.results_url) - success = errors_count + warn_count == 0 - comment = section_title('clang-tidy', success, present) - if not success: - comment += 'clang-tidy found {} errors and {} warnings. ' \ - '{} of them are added as review comments why?.'.format( - self.clang_tidy_result, errors_count, warn_count, inline_comments, - 'https://github.com/google/llvm-premerge-checks/blob/master/docs/clang_tidy.md#review-comments') - - self.comments.append(comment) - self.success = success and self.success - - def add_test_results(self): - """Populates results from build test results XML. - - Only reporting failed tests as the full test suite is too large to upload. - """ - - success = True - present = (self.test_result_file is not None) and os.path.exists( - os.path.join(self.results_dir, self.test_result_file)) - if not present: - print('Warning: Could not find test results file: {}'.format(self.test_result_file)) - self.comments.append(section_title('Unit tests', False, present)) - return - - root_node = etree.parse(os.path.join(self.results_dir, self.test_result_file)) - for test_case in root_node.xpath('//testcase'): - test_result = _test_case_status(test_case) - self.test_stats[test_result] += 1 - - if test_result == 'fail': - success = False - failure = test_case.find('failure') - test_result = { - 'name': test_case.attrib['name'], - 'namespace': test_case.attrib['classname'], - 'result': test_result, - 'duration': float(test_case.attrib['time']), - 'details': failure.text - } - self.unit.append(test_result) - - comment = section_title('Unit tests', success, True) - comment += '{} tests passed, {} failed and {} were skipped.
'.format( - self.test_stats['pass'], - self.test_stats['fail'], - self.test_stats['skip'], - ) - if not success: - comment += 'Failures:
' - for test_case in self.unit: - if test_case['result'] == 'fail': - comment += '{}/{}
'.format(test_case['namespace'], test_case['name']) - self.comments.append(comment) - self.success = success and self.success - - def add_links_to_artifacts(self): - """Comment on a diff, read text from file.""" - file_links = [] - for f in os.listdir(self.results_dir): - if f == 'summary.html': - continue - if f == 'console-log.txt': - self.api.add_artifact(self.ph_id, f, 'build log ' + self.name, self.results_url) - p = os.path.join(self.results_dir, f) - if not os.path.isfile(p): - continue - if os.stat(p).st_size == 0: - continue - file_links.append('{0}'.format(f)) - if len(file_links) > 0: - self.comments.append('Build artifacts:
' + '
'.join(file_links)) - - -def _test_case_status(test_case) -> str: - """Get the status of a test case based on an etree node.""" - if test_case.find('failure') is not None: - return 'fail' - if test_case.find('skipped') is not None: - return 'skip' - return 'pass' - - -def section_title(title: str, ok: bool, present: bool) -> str: - result = 'unknown' - c = '' - if present: - c = 'success' if ok else 'failure' - result = 'pass' if ok else 'fail' - return '{} {}. '.format(title, c, result) - - def _try_call(call): """Tries to call function several times retrying on socked.timeout.""" c = 0 @@ -536,43 +199,3 @@ def _try_call(call): print('Connection to Pharicator failed, retrying: {}'.format(e)) time.sleep(c * 10) break - - -def main(): - parser = argparse.ArgumentParser( - description='Write build status back to Phabricator.') - parser.add_argument('ph_id', type=str) - parser.add_argument('diff_id', type=str) - parser.add_argument('--test-result-file', type=str, dest='test_result_file', default='test-results.xml') - parser.add_argument('--conduit-token', type=str, dest='conduit_token', required=True) - parser.add_argument('--host', type=str, dest='host', default="https://reviews.llvm.org/api/", - help="full URL to API with trailing slash, e.g. https://reviews.llvm.org/api/") - parser.add_argument('--dryrun', action='store_true', - help="output results to the console, do not report back to the server") - parser.add_argument('--buildresult', type=str, default=None, choices=['SUCCESS', 'UNSTABLE', 'FAILURE', 'null']) - parser.add_argument('--clang-format-patch', type=str, default=None, - dest='clang_format_patch', - help="path to diff produced by git-clang-format, relative to results-dir") - parser.add_argument('--clang-tidy-result', type=str, default=None, - dest='clang_tidy_result', - help="path to diff produced by git-clang-tidy, relative to results-dir") - parser.add_argument('--clang-tidy-ignore', type=str, default=None, - dest='clang_tidy_ignore', - help="path to file with patters to exclude commenting on for clang-tidy findings") - parser.add_argument('--results-dir', type=str, default=None, required=True, - dest='results_dir', - help="directory of all build artifacts") - parser.add_argument('--results-url', type=str, default=None, - dest='results_url', - help="public URL to access results directory") - parser.add_argument('--workspace', type=str, required=True, help="path to workspace") - parser.add_argument('--failures', type=str, default=None, help="optional failure messages separated by newline") - parser.add_argument('--name', type=str, default='', help="optional name of the build bot") - args = parser.parse_args() - - reporter = BuildReport(args) - reporter.final_report() - - -if __name__ == '__main__': - main() diff --git a/scripts/run_buildkite.ps1 b/scripts/run_buildkite.ps1 deleted file mode 100644 index 6e97870..0000000 --- a/scripts/run_buildkite.ps1 +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -. ${PSScriptRoot}\common.ps1 - -Write-Output "--- CMake" -& "${PSScriptRoot}\run_cmake.ps1" - -Write-Output "--- ninja all" -& "${PSScriptRoot}\run_ninja.ps1" all - -Write-Output "--- ninja check-all" -& "${PSScriptRoot}\run_ninja.ps1" check-all - -Write-Output "--- done" \ No newline at end of file diff --git a/scripts/run_buildkite.sh b/scripts/run_buildkite.sh deleted file mode 100755 index cb3454f..0000000 --- a/scripts/run_buildkite.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eux - -#folder where this script is stored. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -# dirty workarounds to reuse old scripts... -export WORKSPACE=`pwd` -export TARGET_DIR=/tmp - -# create a clean build folder -BUILD_DIR=${WORKSPACE}/build -rm -rf ${BUILD_DIR} || true -mkdir -p ${BUILD_DIR} - -echo "--- CMake" -${DIR}/run_cmake.sh - -echo "--- ninja all" -${DIR}/run_ninja.sh all - -echo "--- ninja check-all" -${DIR}/run_ninja.sh check-all - -echo "--- done" \ No newline at end of file diff --git a/scripts/run_cmake.ps1 b/scripts/run_cmake.ps1 deleted file mode 100644 index 3a88cb4..0000000 --- a/scripts/run_cmake.ps1 +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -param ( - [Parameter(Mandatory=$false)][string]$projects="default" -) - -. ${PSScriptRoot}\common.ps1 - -# set LLVM_ENABLE_PROJECTS to default value -# if -DetectProjects is set the projects are detected based on the files -# that were modified in the working copy -if ($projects -eq "default") { - # These are the default projects for windows - $LLVM_ENABLE_PROJECTS="clang;clang-tools-extra;libcxx;libc;lld;mlir;libcxxabi" -} elseif ($projects -eq "detect") { - $LLVM_ENABLE_PROJECTS = (git diff HEAD~1 | python ${PSScriptRoot}\choose_projects.py . ) | Out-String - $LLVM_ENABLE_PROJECTS = $LLVM_ENABLE_PROJECTS.replace("`n","").replace("`r","") - if ($LLVM_ENABLE_PROJECTS -eq "") { - Write-Error "Error detecting the affected projects." - exit 1 - } -} else { - $LLVM_ENABLE_PROJECTS=$projects -} - -Write-Output "Setting LLVM_ENABLE_PROJECTS=${LLVM_ENABLE_PROJECTS}" - -# Delete and re-create build folder -Remove-Item build -Recurse -ErrorAction Ignore -New-Item -ItemType Directory -Force -Path build | Out-Null -Push-Location build - -# load Vistual Studio environment variables -Invoke-CmdScript "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64 - -# Make sure we're using the Vistual Studio compiler and linker -$env:CC="cl" -$env:CXX="cl" -$env:LD="link" - -# call CMake -$ErrorActionPreference="Continue" -Invoke-Call -ScriptBlock { - cmake ..\llvm -G Ninja -DCMAKE_BUILD_TYPE=Release ` - -D LLVM_ENABLE_PROJECTS="${LLVM_ENABLE_PROJECTS}" ` - -D LLVM_ENABLE_ASSERTIONS=ON ` - -DLLVM_LIT_ARGS="-v --xunit-xml-output test-results.xml" ` - -D LLVM_ENABLE_DIA_SDK=OFF -} - -# LLVM_ENABLE_DIA_SDK=OFF is a workaround to make the tests pass. -# see https://bugs.llvm.org/show_bug.cgi?id=44151 - -Pop-Location \ No newline at end of file diff --git a/scripts/run_cmake.sh b/scripts/run_cmake.sh deleted file mode 100755 index fa5f17c..0000000 --- a/scripts/run_cmake.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -eux - -# Runs Cmake. -# Inputs: CCACHE_DIR, WORKSPACE, TARGET_DIR; $WORKSPACE/build must exist. -# Outputs: $TARGET_DIR/CMakeCache.txt, $WORKSPACE/compile_commands.json (symlink). - -echo "Running CMake... ======================================" -export CC=clang -export CXX=clang++ -export LD=LLD - -cd "$WORKSPACE"/build -set +e -cmake -GNinja ../llvm -DCMAKE_BUILD_TYPE=Release -D LLVM_ENABLE_LLD=ON \ - -D LLVM_ENABLE_PROJECTS="clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;flang" \ - -D LLVM_CCACHE_BUILD=ON -D LLVM_CCACHE_DIR="${CCACHE_DIR}" -D LLVM_CCACHE_MAXSIZE=20G \ - -D LLVM_ENABLE_ASSERTIONS=ON -DCMAKE_CXX_FLAGS=-gmlt \ - -DLLVM_LIT_ARGS="-v --xunit-xml-output ${WORKSPACE}/build/test-results.xml" -RETURN_CODE="${PIPESTATUS[0]}" -set -e - -rm -f "$WORKSPACE/compile_commands.json" -ln -s "$WORKSPACE"/build/compile_commands.json "$WORKSPACE" -cp CMakeCache.txt ${TARGET_DIR} - -echo "CMake completed ======================================" -exit "${RETURN_CODE}" diff --git a/scripts/run_ninja.ps1 b/scripts/run_ninja.ps1 deleted file mode 100644 index ddd0659..0000000 --- a/scripts/run_ninja.ps1 +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -param( - [Parameter(Mandatory=$true)][string]$target -) - -. ${PSScriptRoot}\common.ps1 - -# cd into build folder -Push-Location build - -# load Visual Studio environment variables -Invoke-CmdScript "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64 - -# call ninja -$PSDefaultParameterValues['*:ErrorAction']='Continue' -Invoke-Call -ScriptBlock {ninja $target} - -Pop-Location diff --git a/scripts/run_ninja.sh b/scripts/run_ninja.sh deleted file mode 100755 index 07eee8f..0000000 --- a/scripts/run_ninja.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# Copyright 2019 Google LLC -# -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://llvm.org/LICENSE.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -eu - -# Runs ninja -# Inputs: TARGET_DIR, WORKSPACE. -# Outputs: $TARGET_DIR/test_results.xml - -CMD=$1 -echo "Running ninja ${CMD}... =====================================" - -ulimit -n 8192 -cd "${WORKSPACE}/build" - -set +e -ninja ${CMD} -RETURN_CODE="$?" -set -e - -echo "ninja ${CMD} completed ======================================" -if test -f "test-results.xml" ; then - echo "copying test_results.xml to ${TARGET_DIR}" - # wait for file? - sleep 10s - du "test-results.xml" - cp test-results.xml "${TARGET_DIR}" - sleep 10s -fi - -exit ${RETURN_CODE} \ No newline at end of file diff --git a/scripts/windows_agent_start_jenkins.ps1 b/scripts/windows_agent_start_jenkins.ps1 deleted file mode 100644 index 69b4e90..0000000 --- a/scripts/windows_agent_start_jenkins.ps1 +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2019 Google LLC - -# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# https://llvm.org/LICENSE.txt - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Pull and start the Docker container for a Windows agent. -# To setup a Windows agent see docs/playbooks.md - -param( - [string]$version = "latest", - [switch]$testing = $false -) - -$NAME="agent-windows-jenkins" -$IMAGE="gcr.io/llvm-premerge-checks/${NAME}:${version}" - -Write-Output "Authenticating docker..." -Write-Output "y`n" | gcloud auth configure-docker - -Write-Output "Pulling new image..." -docker pull ${IMAGE} - -Write-Output "Stopping old container..." -docker stop ${NAME} -docker rm ${NAME} - -Write-Output "Starting container..." -if (${testing}) { - docker run -it ` - -v D:\:C:\ws ` - -v C:\credentials:C:\credentials ` - -e PARENT_HOSTNAME=$env:computername ` - --restart unless-stopped ` - --name ${NAME} ` - ${IMAGE} powershell -} else { - docker run -d ` - -v D:\:C:\ws ` - -v C:\credentials:C:\credentials ` - -e PARENT_HOSTNAME=$env:computername ` - --restart unless-stopped ` - --name ${NAME} ` - ${IMAGE} -}