revi-archive/llvm-premerge-checks
1
0
Fork 0
Code Activity

Merge pull request #222 from google/custom-constraints

Browse source 
custom agent constraints
This commit is contained in:
Mikhail Goncharov 2020-07-24 18:07:06 +02:00 committed by GitHub
commit d0a78f3c8b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 37 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
kubernetes/windows_agent_create.sh
View file

@ -28,8 +28,8 @@ NAME=$1
gcloud beta compute instances create "${NAME}" \
--project="${GCP_PROJECT}" \
--zone="${GCP_ZONE}" \
--machine-type=n1-standard-16 \
--machine-type=n1-standard-32 \
--local-ssd=device-name=local-ssd-0 \
--image=windows-server-2019-dc-for-containers-v20200609 \
--image=windows-server-2019-dc-for-containers-v20200714 \
--image-project=windows-cloud \
--boot-disk-size=100GB

16
scripts/buildkite/analyze_jobs.ipynb
View file

@ -191,13 +191,15 @@
"metadata": {},
"outputs": [],
"source": [
"ds = jobs[jobs['pipeline'] == 'premerge-checks'][jobs['step_key'] == 'windows'][jobs['state']=='passed'][~jobs['agent_name'].str.startswith('buildkite-')]"
"ds = jobs[jobs['pipeline'] == 'premerge-checks'][jobs['step_key'] == 'windows'][jobs['state']=='passed'][~jobs['agent_name'].str.startswith('buildkite-')][jobs['started_at'] > np.datetime64('2020-01-22')]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"ds"
@ -206,12 +208,14 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"scrolled": false
},
"outputs": [],
"source": [
" fig, ax = plt.subplots(figsize=(10,10)) # size of the plot (width, height)\n",
"\n",
" sns.violinplot(\n",
" sns.swarmplot(\n",
" ax=ax,\n",
" x='run_duration',\n",
" y='agent_name',\n",
@ -222,7 +226,9 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"scrolled": false
},
"outputs": [],
"source": [
"t = pd.pivot_table(ds, values=['run_duration'], index=['agent_name'],\n",

15
scripts/buildkite/build_branch_pipeline.py
View file

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import yaml
@ -25,13 +26,17 @@ if __name__ == '__main__':
projects = os.getenv('ph_projects', 'detect')
log_level = os.getenv('ph_log_level', 'WARNING')
steps = []
linux_agents = {'queue': f'{queue_prefix}linux'}
t = os.getenv('ph_linux_agents')
if t is not None:
linux_agents = json.loads(t)
linux_buld_step = {
'label': ':linux: build and test linux',
'key': 'linux',
'commands': [
'set -euo pipefail',
'ccache --clear' if no_cache else '',
'ccache --zero-stats',
'ccache --zero-stats',
'mkdir -p artifacts',
'dpkg -l >> artifacts/packages.txt',
'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks',
@ -54,7 +59,7 @@ if __name__ == '__main__':
'exit \\$EXIT_STATUS',
],
'artifact_paths': ['artifacts/**/*', '*_result.json'],
'agents': {'queue': f'{queue_prefix}linux'},
'agents': linux_agents,
'timeout_in_minutes': 120,
'retry': {'automatic': [
{'exit_status': -1, 'limit': 2}, # Agent lost
@ -64,6 +69,10 @@ if __name__ == '__main__':
clear_sccache = 'powershell -command "sccache --stop-server; ' \
'Remove-Item -Recurse -Force -ErrorAction Ignore $env:SCCACHE_DIR; ' \
'sccache --start-server"'
win_agents = {'queue': f'{queue_prefix}windows'}
t = os.getenv('ph_windows_agents')
if t is not None:
win_agents = json.loads(t)
windows_buld_step = {
'label': ':windows: build and test windows',
'key': 'windows',
@ -88,7 +97,7 @@ if __name__ == '__main__':
'}"',
],
'artifact_paths': ['artifacts/**/*', '*_result.json'],
'agents': {'queue': f'{queue_prefix}windows'},
'agents': win_agents,
'timeout_in_minutes': 120,
'retry': {'automatic': [
{'exit_status': -1, 'limit': 2}, # Agent lost

15
scripts/buildkite/build_master_pipeline.py
View file

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import yaml
@ -25,6 +26,10 @@ if __name__ == '__main__':
log_level = os.getenv('ph_log_level', 'WARNING')
notify_emails = list(filter(None, os.getenv('ph_notify_emails', '').split(',')))
steps = []
linux_agents = {'queue': f'{queue_prefix}linux'}
t = os.getenv('ph_linux_agents')
if t is not None:
linux_agents = json.loads(t)
linux_buld_step = {
'label': ':linux: build and test linux',
'key': 'linux',
@ -48,7 +53,7 @@ if __name__ == '__main__':
'exit \\$EXIT_STATUS',
],
'artifact_paths': ['artifacts/**/*', '*_result.json'],
'agents': {'queue': f'{queue_prefix}linux'},
'agents': linux_agents,
'timeout_in_minutes': 120,
'retry': {'automatic': [
{'exit_status': -1, 'limit': 2}, # Agent lost
@ -58,8 +63,12 @@ if __name__ == '__main__':
clear_sccache = 'powershell -command "sccache --stop-server; ' \
'Remove-Item -Recurse -Force -ErrorAction Ignore $env:SCCACHE_DIR; ' \
'sccache --start-server"'
# FIXME: openmp is removed as it constantly fails.
# FIXME: openmp is removed as it constantly fails. Make this project list be evaluated through "choose_projects".
projects = os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly')
win_agents = {'queue': f'{queue_prefix}windows'}
t = os.getenv('ph_windows_agents')
if t is not None:
win_agents = json.loads(t)
windows_buld_step = {
'label': ':windows: build and test windows',
'key': 'windows',
@ -85,7 +94,7 @@ if __name__ == '__main__':
'}"',
],
'artifact_paths': ['artifacts/**/*', '*_result.json'],
'agents': {'queue': f'{queue_prefix}windows'},
'agents': win_agents,
'timeout_in_minutes': 120,
'retry': {'automatic': [
{'exit_status': -1, 'limit': 2}, # Agent lost