diff --git a/k8s_config b/k8s_config index aab3320..467967a 100644 --- a/k8s_config +++ b/k8s_config @@ -3,4 +3,5 @@ GCP_ZONE="us-central1-a" GCP_CLUSTER="llvm-premerge-checks" GCP_PROJECT="llvm-premerge-checks" GCR_HOSTNAME="gcr.io" -GCS_BUCKET='llvm-premerge-checks' \ No newline at end of file +GCS_BUCKET='llvm-premerge-checks' +GCP_CLUSTER_WINDOWS="windows-agents" diff --git a/kubernetes/cluster_create.sh b/kubernetes/cluster_create.sh index 397af2c..ec46d1d 100755 --- a/kubernetes/cluster_create.sh +++ b/kubernetes/cluster_create.sh @@ -31,8 +31,16 @@ gcloud container clusters create $GCP_CLUSTER --zone $GCP_ZONE \ gcloud container node-pools create jenkins-agents --cluster $GCP_CLUSTER --zone $GCP_ZONE \ --machine-type=n1-standard-32 --num-nodes=2 --local-ssd-count=1 +# created separate cluster for windows, as we need "ip-alias" enabled +# this can't be changed in a running cluster... +gcloud beta container clusters create $GCP_CLUSTER_WINDOWS \ + --enable-ip-alias \ + --num-nodes=1 \ + --release-channel=rapid \ + --enable-private-nodes + # Windows agents with local ssd -gcloud container node-pools create windows-pool --cluster $GCP_CLUSTER \ +gcloud container node-pools create windows-pool --cluster $GCP_CLUSTER_WINDOWS \ --image-type=WINDOWS_SAC --no-enable-autoupgrade \ --machine-type=n1-standard-16 --local-ssd-count=1 diff --git a/kubernetes/jenkins.yaml b/kubernetes/jenkins.yaml index ba2b9be..d295b5c 100644 --- a/kubernetes/jenkins.yaml +++ b/kubernetes/jenkins.yaml @@ -186,44 +186,5 @@ spec: # directory location on host path: /mnt/disks/ssd0 type: Directory - nodeSelector: - cloud.google.com/gke-nodepool: jenkins-agents ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: agent-windows-visualstudio2017-ssd - namespace: jenkins -spec: - replicas: 1 - template: - metadata: - labels: - app: agent-windows-visualstudio2017-ssd - spec: - containers: - - name: agent-windows-visualstudio2017-ssd - image: gcr.io/llvm-premerge-checks/agent-windows - resources: - limits: - cpu: 14 - memory: 20Gi - requests: - cpu: 14 - memory: 20Gi -# volumeMounts: -# - name: nfs-pvc -# mountPath: /mnt/nfs -# - name: ssd -# mountPath: /mnt/disks/ssd0 -# volumes: -# - name: nfs-pvc -# persistentVolumeClaim: -# claimName: nfs-jenkins -# - name: ssd -# hostPath: -# # directory location on host -# path: /mnt/disks/ssd0 -# type: Directory nodeSelector: cloud.google.com/gke-nodepool: jenkins-agents \ No newline at end of file diff --git a/kubernetes/windows-agents/Deployment.yaml b/kubernetes/windows-agents/Deployment.yaml new file mode 100644 index 0000000..4e00384 --- /dev/null +++ b/kubernetes/windows-agents/Deployment.yaml @@ -0,0 +1,58 @@ +# Copyright 2019 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agent-windows-visualstudio2017-ssd + labels: + app: agent-windows-visualstudio2017-ssd +spec: + replicas: 1 + selector: + matchLabels: + app: agent-windows-visualstudio2017-ssd + template: + metadata: + labels: + app: agent-windows-visualstudio2017-ssd + spec: + containers: + - name: agent-windows-visualstudio2017-ssd + image: gcr.io/llvm-premerge-checks/agent-windows-jenkins + resources: + limits: + cpu: 14 + memory: 20Gi + requests: + cpu: 14 + memory: 20Gi +# volumeMounts: +# - name: nfs-pvc +# mountPath: /mnt/nfs +# - name: ssd +# mountPath: /mnt/disks/ssd0 +# volumes: +# - name: nfs-pvc +# persistentVolumeClaim: +# claimName: nfs-jenkins +# - name: ssd +# hostPath: +# # directory location on host +# path: /mnt/disks/ssd0 +# type: Directory + nodeSelector: + kubernetes.io/os: windows + cloud.google.com/gke-nodepool: windows-pool \ No newline at end of file diff --git a/kubernetes/windows-agents/kustomization.yaml b/kubernetes/windows-agents/kustomization.yaml new file mode 100644 index 0000000..521cb26 --- /dev/null +++ b/kubernetes/windows-agents/kustomization.yaml @@ -0,0 +1,19 @@ +# Copyright 2019 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - Deployment.yaml + diff --git a/scripts/choose_projects.py b/scripts/choose_projects.py index 9d7804d..268281a 100755 --- a/scripts/choose_projects.py +++ b/scripts/choose_projects.py @@ -41,6 +41,7 @@ class ChooseProjects: def __init__(self, llvm_dir: str): self.llvm_dir = llvm_dir + self.dependencies = dict() # type: Dict[str,List[str]] self.usages = dict() # type: Dict[str,List[str]] self.all_projects = [] # type: List[str] self.excluded_projects = set() # type: Set[str] @@ -51,7 +52,8 @@ class ChooseProjects: logging.info('loading project config from {}'.format(self.DEPENDENCIES_FILE)) with open(self.DEPENDENCIES_FILE) as dependencies_file: config = yaml.load(dependencies_file, Loader=yaml.SafeLoader) - for user, used_list in config['dependencies'].items(): + self.dependencies = config['dependencies'] + for user, used_list in self.dependencies.items(): for used in used_list: self.usages.setdefault(used,[]).append(user) self.all_projects = config['allprojects'] @@ -72,6 +74,7 @@ class ChooseProjects: return 0 affected_projects = self.get_affected_projects(changed_projects) + affected_projects = self.add_dependencies(affected_projects) affected_projects = affected_projects - self.excluded_projects print(';'.join(sorted(affected_projects))) return 0 @@ -132,6 +135,23 @@ class ChooseProjects: return affected_projects + def add_dependencies(self, projects: Set[str]): + """Return projects and their dependencies. + + All all dependencies to `projects` so that they can be built. + """ + result = set(projects) + last_len = -1 + while len(result) != last_len: + last_len = len(result) + changes = set() + for project in result: + if project in self.dependencies: + changes.update(self.dependencies[project]) + print(changes) + result.update(changes) + return result + if __name__ == "__main__": logging.basicConfig(filename='choose_projects.log', level=logging.INFO) parser = argparse.ArgumentParser(description='Compute the projects affected by a change.')