diff --git a/kubernetes/windows_agent_create.sh b/kubernetes/windows_agent_create.sh index 3ac26f6..d34e425 100755 --- a/kubernetes/windows_agent_create.sh +++ b/kubernetes/windows_agent_create.sh @@ -28,8 +28,8 @@ NAME=$1 gcloud beta compute instances create "${NAME}" \ --project="${GCP_PROJECT}" \ --zone="${GCP_ZONE}" \ - --machine-type=n1-standard-16 \ + --machine-type=n1-standard-32 \ --local-ssd=device-name=local-ssd-0 \ - --image=windows-server-2019-dc-for-containers-v20200609 \ + --image=windows-server-2019-dc-for-containers-v20200714 \ --image-project=windows-cloud \ --boot-disk-size=100GB diff --git a/scripts/buildkite/analyze_jobs.ipynb b/scripts/buildkite/analyze_jobs.ipynb index 79b6bc6..e37a76c 100644 --- a/scripts/buildkite/analyze_jobs.ipynb +++ b/scripts/buildkite/analyze_jobs.ipynb @@ -191,13 +191,15 @@ "metadata": {}, "outputs": [], "source": [ - "ds = jobs[jobs['pipeline'] == 'premerge-checks'][jobs['step_key'] == 'windows'][jobs['state']=='passed'][~jobs['agent_name'].str.startswith('buildkite-')]" + "ds = jobs[jobs['pipeline'] == 'premerge-checks'][jobs['step_key'] == 'windows'][jobs['state']=='passed'][~jobs['agent_name'].str.startswith('buildkite-')][jobs['started_at'] > np.datetime64('2020-01-22')]" ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "ds" @@ -206,12 +208,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": false + }, "outputs": [], "source": [ " fig, ax = plt.subplots(figsize=(10,10)) # size of the plot (width, height)\n", "\n", - " sns.violinplot(\n", + " sns.swarmplot(\n", " ax=ax,\n", " x='run_duration',\n", " y='agent_name',\n", @@ -222,7 +226,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": false + }, "outputs": [], "source": [ "t = pd.pivot_table(ds, values=['run_duration'], index=['agent_name'],\n", diff --git a/scripts/buildkite/build_branch_pipeline.py b/scripts/buildkite/build_branch_pipeline.py index 3e7c081..3059bac 100755 --- a/scripts/buildkite/build_branch_pipeline.py +++ b/scripts/buildkite/build_branch_pipeline.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import yaml @@ -25,13 +26,17 @@ if __name__ == '__main__': projects = os.getenv('ph_projects', 'detect') log_level = os.getenv('ph_log_level', 'WARNING') steps = [] + linux_agents = {'queue': f'{queue_prefix}linux'} + t = os.getenv('ph_linux_agents') + if t is not None: + linux_agents = json.loads(t) linux_buld_step = { 'label': ':linux: build and test linux', 'key': 'linux', 'commands': [ 'set -euo pipefail', 'ccache --clear' if no_cache else '', - 'ccache --zero-stats', + 'ccache --zero-stats', 'mkdir -p artifacts', 'dpkg -l >> artifacts/packages.txt', 'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks', @@ -54,7 +59,7 @@ if __name__ == '__main__': 'exit \\$EXIT_STATUS', ], 'artifact_paths': ['artifacts/**/*', '*_result.json'], - 'agents': {'queue': f'{queue_prefix}linux'}, + 'agents': linux_agents, 'timeout_in_minutes': 120, 'retry': {'automatic': [ {'exit_status': -1, 'limit': 2}, # Agent lost @@ -64,6 +69,10 @@ if __name__ == '__main__': clear_sccache = 'powershell -command "sccache --stop-server; ' \ 'Remove-Item -Recurse -Force -ErrorAction Ignore $env:SCCACHE_DIR; ' \ 'sccache --start-server"' + win_agents = {'queue': f'{queue_prefix}windows'} + t = os.getenv('ph_windows_agents') + if t is not None: + win_agents = json.loads(t) windows_buld_step = { 'label': ':windows: build and test windows', 'key': 'windows', @@ -88,7 +97,7 @@ if __name__ == '__main__': '}"', ], 'artifact_paths': ['artifacts/**/*', '*_result.json'], - 'agents': {'queue': f'{queue_prefix}windows'}, + 'agents': win_agents, 'timeout_in_minutes': 120, 'retry': {'automatic': [ {'exit_status': -1, 'limit': 2}, # Agent lost diff --git a/scripts/buildkite/build_master_pipeline.py b/scripts/buildkite/build_master_pipeline.py index ab28efc..cc5dd0a 100755 --- a/scripts/buildkite/build_master_pipeline.py +++ b/scripts/buildkite/build_master_pipeline.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import yaml @@ -25,6 +26,10 @@ if __name__ == '__main__': log_level = os.getenv('ph_log_level', 'WARNING') notify_emails = list(filter(None, os.getenv('ph_notify_emails', '').split(','))) steps = [] + linux_agents = {'queue': f'{queue_prefix}linux'} + t = os.getenv('ph_linux_agents') + if t is not None: + linux_agents = json.loads(t) linux_buld_step = { 'label': ':linux: build and test linux', 'key': 'linux', @@ -48,7 +53,7 @@ if __name__ == '__main__': 'exit \\$EXIT_STATUS', ], 'artifact_paths': ['artifacts/**/*', '*_result.json'], - 'agents': {'queue': f'{queue_prefix}linux'}, + 'agents': linux_agents, 'timeout_in_minutes': 120, 'retry': {'automatic': [ {'exit_status': -1, 'limit': 2}, # Agent lost @@ -58,8 +63,12 @@ if __name__ == '__main__': clear_sccache = 'powershell -command "sccache --stop-server; ' \ 'Remove-Item -Recurse -Force -ErrorAction Ignore $env:SCCACHE_DIR; ' \ 'sccache --start-server"' - # FIXME: openmp is removed as it constantly fails. + # FIXME: openmp is removed as it constantly fails. Make this project list be evaluated through "choose_projects". projects = os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly') + win_agents = {'queue': f'{queue_prefix}windows'} + t = os.getenv('ph_windows_agents') + if t is not None: + win_agents = json.loads(t) windows_buld_step = { 'label': ':windows: build and test windows', 'key': 'windows', @@ -85,7 +94,7 @@ if __name__ == '__main__': '}"', ], 'artifact_paths': ['artifacts/**/*', '*_result.json'], - 'agents': {'queue': f'{queue_prefix}windows'}, + 'agents': win_agents, 'timeout_in_minutes': 120, 'retry': {'automatic': [ {'exit_status': -1, 'limit': 2}, # Agent lost