feat: Add anti-affinity labels to ensure 1 GPU-enabled pod per node

Due to the lack of support for GPU virtualization by Weaviate (see weaviate/t2v-transformers-models#39 for details), we need to ensure each pod that uses the GPU, utilizes the full pod without sharing it. The way to ensure this is to schedule each GPU enabled pod to its own dedicated node using Kubernetes's anti-affinity feature. This addresses this to make it easy to run clusters with GPU support.
uhadmin · Dec 29, 2022 · bfefa65 · bfefa65
1 parent 08e3d5e
commit bfefa65
Show file tree

Hide file tree

Showing 9 changed files with 92 additions and 1 deletion.
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,5 @@
 weaviate/charts
-weaviate/weaviate-*.tgz
+#weaviate/weaviate-*.tgz
 weaviate/weaviate.tgz
 weaviate/out.yml
 tmp/*

diff --git a/weaviate/templates/contextionaryDeployment.yaml b/weaviate/templates/contextionaryDeployment.yaml
@@ -6,6 +6,7 @@ metadata:
   labels:
     name: contextionary
     app: contextionary
+    requiresSoleAccessToGpu: 'true'
 spec:
   replicas: {{ index .Values "modules" "text2vec-contextionary" "replicas" }}
   selector:
@@ -16,6 +17,18 @@ spec:
       labels:
         app: contextionary
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: requiresSoleAccessToGpu
+                  operator: In
+                  values:
+                  - 'true'
+              topologyKey: kubernetes.io/hostname
       containers:
       - name: contextionary
         image: {{ index .Values "modules" "text2vec-contextionary" "registry" }}/{{ index .Values "modules" "text2vec-contextionary" "repo" }}:{{ index .Values "modules" "text2vec-contextionary" "tag" }}

diff --git a/weaviate/templates/img2vecNeuralDeployment.yaml b/weaviate/templates/img2vecNeuralDeployment.yaml
@@ -6,6 +6,7 @@ metadata:
   labels:
     name: {{ index .Values "modules" "img2vec-neural" "fullnameOverride" }}
     app: {{ index .Values "modules" "img2vec-neural" "fullnameOverride" }}
+    requiresSoleAccessToGpu: 'true'
 spec:
   replicas: {{ index .Values "modules" "img2vec-neural" "replicas" }}
   selector:
@@ -16,6 +17,18 @@ spec:
       labels:
         app: {{ index .Values "modules" "img2vec-neural" "fullnameOverride" }}
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: requiresSoleAccessToGpu
+                  operator: In
+                  values:
+                  - 'true'
+              topologyKey: kubernetes.io/hostname
       containers:
       - name: {{ index .Values "modules" "img2vec-neural" "fullnameOverride" }}
         image: {{ index .Values "modules" "img2vec-neural" "registry" }}/{{ index .Values "modules" "img2vec-neural" "repo" }}:{{ index .Values "modules" "img2vec-neural" "tag" }}

diff --git a/weaviate/templates/multi2vecClipDeployment.yaml b/weaviate/templates/multi2vecClipDeployment.yaml
@@ -6,6 +6,7 @@ metadata:
   labels:
     name: {{ index .Values "modules" "multi2vec-clip" "fullnameOverride" }}
     app: {{ index .Values "modules" "multi2vec-clip" "fullnameOverride" }}
+    requiresSoleAccessToGpu: 'true'
 spec:
   replicas: {{ index .Values "modules" "multi2vec-clip" "replicas" }}
   selector:
@@ -16,6 +17,18 @@ spec:
       labels:
         app: {{ index .Values "modules" "multi2vec-clip" "fullnameOverride" }}
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: requiresSoleAccessToGpu
+                  operator: In
+                  values:
+                  - 'true'
+              topologyKey: kubernetes.io/hostname
       containers:
       - name: {{ index .Values "modules" "multi2vec-clip" "fullnameOverride" }}
         image: {{ index .Values "modules" "multi2vec-clip" "registry" }}/{{ index .Values "modules" "multi2vec-clip" "repo" }}:{{ index .Values "modules" "multi2vec-clip" "tag" }}

diff --git a/weaviate/templates/nerTransformersDeployment.yaml b/weaviate/templates/nerTransformersDeployment.yaml
@@ -6,6 +6,7 @@ metadata:
   labels:
     name: {{ index .Values "modules" "ner-transformers" "fullnameOverride" }}
     app: {{ index .Values "modules" "ner-transformers" "fullnameOverride" }}
+    requiresSoleAccessToGpu: 'true'
 spec:
   replicas: {{ index .Values "modules" "ner-transformers" "replicas" }}
   selector:
@@ -16,6 +17,18 @@ spec:
       labels:
         app: {{ index .Values "modules" "ner-transformers" "fullnameOverride" }}
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: requiresSoleAccessToGpu
+                  operator: In
+                  values:
+                  - 'true'
+              topologyKey: kubernetes.io/hostname
       containers:
       - name: {{ index .Values "modules" "ner-transformers" "fullnameOverride" }}
         image: {{ index .Values "modules" "ner-transformers" "registry" }}/{{ index .Values "modules" "ner-transformers" "repo" }}:{{ index .Values "modules" "ner-transformers" "tag" }}

diff --git a/weaviate/templates/qnaTransformersDeployment.yaml b/weaviate/templates/qnaTransformersDeployment.yaml
@@ -6,6 +6,7 @@ metadata:
   labels:
     name: {{ index .Values "modules" "qna-transformers" "fullnameOverride" }}
     app: {{ index .Values "modules" "qna-transformers" "fullnameOverride" }}
+    requiresSoleAccessToGpu: 'true'
 spec:
   replicas: {{ index .Values "modules" "qna-transformers" "replicas" }}
   selector:
@@ -16,6 +17,18 @@ spec:
       labels:
         app: {{ index .Values "modules" "qna-transformers" "fullnameOverride" }}
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: requiresSoleAccessToGpu
+                  operator: In
+                  values:
+                  - 'true'
+              topologyKey: kubernetes.io/hostname
       containers:
       - name: {{ index .Values "modules" "qna-transformers" "fullnameOverride" }}
         image: {{ index .Values "modules" "qna-transformers" "registry" }}/{{ index .Values "modules" "qna-transformers" "repo" }}:{{ index .Values "modules" "qna-transformers" "tag" }}

diff --git a/weaviate/templates/sumTransformersDeployment.yaml b/weaviate/templates/sumTransformersDeployment.yaml
@@ -6,6 +6,7 @@ metadata:
   labels:
     name: {{ index .Values "modules" "sum-transformers" "fullnameOverride" }}
     app: {{ index .Values "modules" "sum-transformers" "fullnameOverride" }}
+    requiresSoleAccessToGpu: 'true'
 spec:
   replicas: {{ index .Values "modules" "sum-transformers" "replicas" }}
   selector:
@@ -16,6 +17,18 @@ spec:
       labels:
         app: {{ index .Values "modules" "sum-transformers" "fullnameOverride" }}
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: requiresSoleAccessToGpu
+                  operator: In
+                  values:
+                  - 'true'
+              topologyKey: kubernetes.io/hostname
       containers:
       - name: {{ index .Values "modules" "sum-transformers" "fullnameOverride" }}
         image: {{ index .Values "modules" "sum-transformers" "registry" }}/{{ index .Values "modules" "sum-transformers" "repo" }}:{{ index .Values "modules" "sum-transformers" "tag" }}

diff --git a/weaviate/templates/transformersInferenceDeployment.yaml b/weaviate/templates/transformersInferenceDeployment.yaml
@@ -6,6 +6,7 @@ metadata:
   labels:
     name: {{ index .Values "modules" "text2vec-transformers" "fullnameOverride" }}
     app: {{ index .Values "modules" "text2vec-transformers" "fullnameOverride" }}
+    requiresSoleAccessToGpu: 'true'
 spec:
   replicas: {{ index .Values "modules" "text2vec-transformers" "replicas" }}
   selector:
@@ -16,6 +17,18 @@ spec:
       labels:
         app: {{ index .Values "modules" "text2vec-transformers" "fullnameOverride" }}
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: requiresSoleAccessToGpu
+                  operator: In
+                  values:
+                  - 'true'
+              topologyKey: kubernetes.io/hostname
       containers:
       - name: {{ index .Values "modules" "text2vec-transformers" "fullnameOverride" }}
         image: {{ index .Values "modules" "text2vec-transformers" "registry" }}/{{ index .Values "modules" "text2vec-transformers" "repo" }}:{{ index .Values "modules" "text2vec-transformers" "tag" }}

diff --git a/weaviate/weaviate-15.2.0.tgz b/weaviate/weaviate-15.2.0.tgz