Merge branch 'master' of ssh://github.com/google/llvm-premerge-checks

k8s_config

@@ -3,4 +3,5 @@ GCP_ZONE="us-central1-a"
 GCP_CLUSTER="llvm-premerge-checks"
 GCP_PROJECT="llvm-premerge-checks"
 GCR_HOSTNAME="gcr.io"
-GCS_BUCKET='llvm-premerge-checks'
+GCS_BUCKET='llvm-premerge-checks'
+GCP_CLUSTER_WINDOWS="windows-agents"

kubernetes/cluster_create.sh

@@ -31,8 +31,16 @@ gcloud container clusters create $GCP_CLUSTER --zone $GCP_ZONE \
 gcloud container node-pools create jenkins-agents --cluster $GCP_CLUSTER --zone $GCP_ZONE \
     --machine-type=n1-standard-32 --num-nodes=2 --local-ssd-count=1
 
+# created separate cluster for windows, as we need "ip-alias" enabled
+# this can't be changed in a running cluster...
+gcloud beta container clusters create $GCP_CLUSTER_WINDOWS \
+  --enable-ip-alias \
+  --num-nodes=1 \
+  --release-channel=rapid \
+  --enable-private-nodes
+
 # Windows agents with local ssd
-gcloud container node-pools create windows-pool --cluster $GCP_CLUSTER \
+gcloud container node-pools create windows-pool --cluster $GCP_CLUSTER_WINDOWS \
     --image-type=WINDOWS_SAC --no-enable-autoupgrade \
     --machine-type=n1-standard-16 --local-ssd-count=1
 

kubernetes/jenkins.yaml

@@ -186,44 +186,5 @@ spec:
           # directory location on host
           path: /mnt/disks/ssd0
           type: Directory          
-      nodeSelector:
-        cloud.google.com/gke-nodepool: jenkins-agents
----
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: agent-windows-visualstudio2017-ssd
-  namespace: jenkins
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: agent-windows-visualstudio2017-ssd
-    spec:
-      containers:
-      - name: agent-windows-visualstudio2017-ssd
-        image: gcr.io/llvm-premerge-checks/agent-windows
-        resources:
-          limits:
-            cpu: 14
-            memory: 20Gi
-          requests:
-            cpu: 14
-            memory: 20Gi
-#        volumeMounts:
-#        - name: nfs-pvc
-#          mountPath: /mnt/nfs          
-#        - name: ssd 
-#          mountPath: /mnt/disks/ssd0
-#      volumes:
-#      - name: nfs-pvc
-#        persistentVolumeClaim:
-#          claimName: nfs-jenkins
-#      - name: ssd
-#        hostPath:
-#          # directory location on host
-#          path: /mnt/disks/ssd0
-#          type: Directory          
       nodeSelector:
         cloud.google.com/gke-nodepool: jenkins-agents

kubernetes/windows-agents/Deployment.yaml

@@ -0,0 +1,58 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: agent-windows-visualstudio2017-ssd
+  labels:
+    app: agent-windows-visualstudio2017-ssd
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: agent-windows-visualstudio2017-ssd
+  template:
+    metadata:
+      labels:
+        app: agent-windows-visualstudio2017-ssd
+    spec:
+      containers:
+      - name: agent-windows-visualstudio2017-ssd
+        image: gcr.io/llvm-premerge-checks/agent-windows-jenkins
+        resources:
+          limits:
+            cpu: 14
+            memory: 20Gi
+          requests:
+            cpu: 14
+            memory: 20Gi
+#        volumeMounts:
+#        - name: nfs-pvc
+#          mountPath: /mnt/nfs          
+#        - name: ssd 
+#          mountPath: /mnt/disks/ssd0
+#      volumes:
+#      - name: nfs-pvc
+#        persistentVolumeClaim:
+#          claimName: nfs-jenkins
+#      - name: ssd
+#        hostPath:
+#          # directory location on host
+#          path: /mnt/disks/ssd0
+#          type: Directory          
+      nodeSelector:
+        kubernetes.io/os: windows
+        cloud.google.com/gke-nodepool: windows-pool

kubernetes/windows-agents/kustomization.yaml

@@ -0,0 +1,19 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://llvm.org/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - Deployment.yaml
+

scripts/choose_projects.py

@@ -41,6 +41,7 @@ class ChooseProjects:
 
     def __init__(self, llvm_dir: str):
         self.llvm_dir = llvm_dir
+        self.dependencies = dict() # type: Dict[str,List[str]]
         self.usages = dict()   # type: Dict[str,List[str]]
         self.all_projects = [] # type: List[str]
         self.excluded_projects = set() # type: Set[str]
@@ -51,7 +52,8 @@ class ChooseProjects:
         logging.info('loading project config from {}'.format(self.DEPENDENCIES_FILE))
         with open(self.DEPENDENCIES_FILE) as dependencies_file:
             config = yaml.load(dependencies_file, Loader=yaml.SafeLoader)
-        for user, used_list in config['dependencies'].items():
+        self.dependencies = config['dependencies']
+        for user, used_list in self.dependencies.items():
             for used in used_list:
                 self.usages.setdefault(used,[]).append(user)
         self.all_projects = config['allprojects']
@@ -72,6 +74,7 @@ class ChooseProjects:
             return 0
 
         affected_projects = self.get_affected_projects(changed_projects)
+        affected_projects = self.add_dependencies(affected_projects)
         affected_projects = affected_projects - self.excluded_projects
         print(';'.join(sorted(affected_projects)))
         return 0
@@ -132,6 +135,23 @@ class ChooseProjects:
         return affected_projects
 
 
+    def add_dependencies(self, projects: Set[str]):
+        """Return projects and their dependencies.
+
+        All all dependencies to `projects` so that they can be built.
+        """
+        result = set(projects)
+        last_len = -1
+        while len(result) != last_len:
+            last_len = len(result)
+            changes = set()
+            for project in result:
+                if project in self.dependencies:
+                    changes.update(self.dependencies[project])
+            print(changes)
+            result.update(changes)
+        return result
+
 if __name__ == "__main__":
     logging.basicConfig(filename='choose_projects.log', level=logging.INFO)
     parser = argparse.ArgumentParser(description='Compute the projects affected by a change.')