From fa116476700cd4cc70e9aae9ad9efb196d265ad7 Mon Sep 17 00:00:00 2001
From: Mikhail Goncharov <goncharov.mikhail@gmail.com>
Date: Tue, 21 Jul 2020 10:26:00 +0200
Subject: [PATCH] retry build step if agent is gone or shut down

---
 scripts/buildkite/build_branch_pipeline.py | 8 ++++++++
 scripts/buildkite/build_master_pipeline.py | 8 ++++++++
 2 files changed, 16 insertions(+)

diff --git a/scripts/buildkite/build_branch_pipeline.py b/scripts/buildkite/build_branch_pipeline.py
index 4611814..0813560 100755
--- a/scripts/buildkite/build_branch_pipeline.py
+++ b/scripts/buildkite/build_branch_pipeline.py
@@ -56,6 +56,10 @@ if __name__ == '__main__':
         'artifact_paths': ['artifacts/**/*', '*_result.json'],
         'agents': {'queue': f'{queue_prefix}linux'},
         'timeout_in_minutes': 120,
+        'retry': {'automatic': [
+            {'exit_status': -1, 'limit': 2},  # Agent lost
+            {'exit_status': 255, 'limit': 2},  # Forced agent shutdown
+        ]},
     }
     clear_sccache = 'powershell -command "sccache --stop-server; ' \
                     'Remove-Item -Recurse -Force -ErrorAction Ignore $env:SCCACHE_DIR; ' \
@@ -86,6 +90,10 @@ if __name__ == '__main__':
         'artifact_paths': ['artifacts/**/*', '*_result.json'],
         'agents': {'queue': f'{queue_prefix}windows'},
         'timeout_in_minutes': 120,
+        'retry': {'automatic': [
+            {'exit_status': -1, 'limit': 2},  # Agent lost
+            {'exit_status': 255, 'limit': 2},  # Forced agent shutdown
+        ]},
     }
     deps = []
     if os.getenv('ph_skip_linux') is None:
diff --git a/scripts/buildkite/build_master_pipeline.py b/scripts/buildkite/build_master_pipeline.py
index 1999c56..808d2e8 100755
--- a/scripts/buildkite/build_master_pipeline.py
+++ b/scripts/buildkite/build_master_pipeline.py
@@ -49,6 +49,10 @@ if __name__ == '__main__':
         'artifact_paths': ['artifacts/**/*', '*_result.json'],
         'agents': {'queue': f'{queue_prefix}linux'},
         'timeout_in_minutes': 120,
+        'retry': {'automatic': [
+            {'exit_status': -1, 'limit': 2},  # Agent lost
+            {'exit_status': 255, 'limit': 2},  # Forced agent shutdown
+        ]},
     }
     clear_sccache = 'powershell -command "sccache --stop-server; ' \
                     'Remove-Item -Recurse -Force -ErrorAction Ignore $env:SCCACHE_DIR; ' \
@@ -82,6 +86,10 @@ if __name__ == '__main__':
         'artifact_paths': ['artifacts/**/*', '*_result.json'],
         'agents': {'queue': f'{queue_prefix}windows'},
         'timeout_in_minutes': 120,
+        'retry': {'automatic': [
+            {'exit_status': -1, 'limit': 2},  # Agent lost
+            {'exit_status': 255, 'limit': 2},  # Forced agent shutdown
+        ]},
     }
     if os.getenv('ph_skip_linux') is None:
         steps.append(linux_buld_step)