Merge pull request #222 from google/custom-constraints

custom agent constraints

kubernetes/windows_agent_create.sh

@@ -28,8 +28,8 @@ NAME=$1
 gcloud beta compute instances create "${NAME}" \
   --project="${GCP_PROJECT}" \
   --zone="${GCP_ZONE}" \
-  --machine-type=n1-standard-16 \
+  --machine-type=n1-standard-32 \
   --local-ssd=device-name=local-ssd-0 \
-  --image=windows-server-2019-dc-for-containers-v20200609 \
+  --image=windows-server-2019-dc-for-containers-v20200714 \
   --image-project=windows-cloud \
   --boot-disk-size=100GB

scripts/buildkite/analyze_jobs.ipynb

@@ -191,13 +191,15 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ds = jobs[jobs['pipeline'] == 'premerge-checks'][jobs['step_key'] == 'windows'][jobs['state']=='passed'][~jobs['agent_name'].str.startswith('buildkite-')]"
+    "ds = jobs[jobs['pipeline'] == 'premerge-checks'][jobs['step_key'] == 'windows'][jobs['state']=='passed'][~jobs['agent_name'].str.startswith('buildkite-')][jobs['started_at'] > np.datetime64('2020-01-22')]"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "scrolled": true
+   },
    "outputs": [],
    "source": [
     "ds"
@@ -206,12 +208,14 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "scrolled": false
+   },
    "outputs": [],
    "source": [
     "  fig, ax = plt.subplots(figsize=(10,10)) # size of the plot (width, height)\n",
     "\n",
-    "  sns.violinplot(\n",
+    "  sns.swarmplot(\n",
     "        ax=ax,\n",
     "        x='run_duration',\n",
     "        y='agent_name',\n",
@@ -222,7 +226,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "scrolled": false
+   },
    "outputs": [],
    "source": [
     "t = pd.pivot_table(ds, values=['run_duration'], index=['agent_name'],\n",

scripts/buildkite/build_branch_pipeline.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
 import yaml
 
@@ -25,13 +26,17 @@ if __name__ == '__main__':
     projects = os.getenv('ph_projects', 'detect')
     log_level = os.getenv('ph_log_level', 'WARNING')
     steps = []
+    linux_agents = {'queue': f'{queue_prefix}linux'}
+    t = os.getenv('ph_linux_agents')
+    if t is not None:
+        linux_agents = json.loads(t)
     linux_buld_step = {
         'label': ':linux: build and test linux',
         'key': 'linux',
         'commands': [
             'set -euo pipefail',
             'ccache --clear' if no_cache else '',
-             'ccache --zero-stats',
+            'ccache --zero-stats',
             'mkdir -p artifacts',
             'dpkg -l >> artifacts/packages.txt',
             'export SRC=${BUILDKITE_BUILD_PATH}/llvm-premerge-checks',
@@ -54,7 +59,7 @@ if __name__ == '__main__':
             'exit \\$EXIT_STATUS',
         ],
         'artifact_paths': ['artifacts/**/*', '*_result.json'],
-        'agents': {'queue': f'{queue_prefix}linux'},
+        'agents': linux_agents,
         'timeout_in_minutes': 120,
         'retry': {'automatic': [
             {'exit_status': -1, 'limit': 2},  # Agent lost
@@ -64,6 +69,10 @@ if __name__ == '__main__':
     clear_sccache = 'powershell -command "sccache --stop-server; ' \
                     'Remove-Item -Recurse -Force -ErrorAction Ignore $env:SCCACHE_DIR; ' \
                     'sccache --start-server"'
+    win_agents = {'queue': f'{queue_prefix}windows'}
+    t = os.getenv('ph_windows_agents')
+    if t is not None:
+        win_agents = json.loads(t)
     windows_buld_step = {
         'label': ':windows: build and test windows',
         'key': 'windows',
@@ -88,7 +97,7 @@ if __name__ == '__main__':
             '}"',
         ],
         'artifact_paths': ['artifacts/**/*', '*_result.json'],
-        'agents': {'queue': f'{queue_prefix}windows'},
+        'agents': win_agents,
         'timeout_in_minutes': 120,
         'retry': {'automatic': [
             {'exit_status': -1, 'limit': 2},  # Agent lost

scripts/buildkite/build_master_pipeline.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
 import yaml
 
@@ -25,6 +26,10 @@ if __name__ == '__main__':
     log_level = os.getenv('ph_log_level', 'WARNING')
     notify_emails = list(filter(None, os.getenv('ph_notify_emails', '').split(',')))
     steps = []
+    linux_agents = {'queue': f'{queue_prefix}linux'}
+    t = os.getenv('ph_linux_agents')
+    if t is not None:
+        linux_agents = json.loads(t)
     linux_buld_step = {
         'label': ':linux: build and test linux',
         'key': 'linux',
@@ -48,7 +53,7 @@ if __name__ == '__main__':
             'exit \\$EXIT_STATUS',
         ],
         'artifact_paths': ['artifacts/**/*', '*_result.json'],
-        'agents': {'queue': f'{queue_prefix}linux'},
+        'agents': linux_agents,
         'timeout_in_minutes': 120,
         'retry': {'automatic': [
             {'exit_status': -1, 'limit': 2},  # Agent lost
@@ -58,8 +63,12 @@ if __name__ == '__main__':
     clear_sccache = 'powershell -command "sccache --stop-server; ' \
                     'Remove-Item -Recurse -Force -ErrorAction Ignore $env:SCCACHE_DIR; ' \
                     'sccache --start-server"'
-    # FIXME: openmp is removed as it constantly fails.
+    # FIXME: openmp is removed as it constantly fails. Make this project list be evaluated through "choose_projects".
     projects = os.getenv('ph_projects', 'clang;clang-tools-extra;libc;libcxx;libcxxabi;lld;libunwind;mlir;polly')
+    win_agents = {'queue': f'{queue_prefix}windows'}
+    t = os.getenv('ph_windows_agents')
+    if t is not None:
+        win_agents = json.loads(t)
     windows_buld_step = {
         'label': ':windows: build and test windows',
         'key': 'windows',
@@ -85,7 +94,7 @@ if __name__ == '__main__':
             '}"',
         ],
         'artifact_paths': ['artifacts/**/*', '*_result.json'],
-        'agents': {'queue': f'{queue_prefix}windows'},
+        'agents': win_agents,
         'timeout_in_minutes': 120,
         'retry': {'automatic': [
             {'exit_status': -1, 'limit': 2},  # Agent lost