Skip to content

Commit

Permalink
Address comment
Browse files Browse the repository at this point in the history
  • Loading branch information
nstogner committed Nov 24, 2024
1 parent 2bfd3ec commit ad72797
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 13 deletions.
2 changes: 1 addition & 1 deletion internal/modelcontroller/adapters.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ const (
)

// reconcileAdapters ensures that the specified adapters are loaded in the model server pods.
// Loaded adapters are identified by the presence of a label with the adapter name and the hash
// Loaded adapters are identified by the presence of a Pod label with the adapter name and the hash
// of the adapter URL.
// At request-time, the endpoint resolver will inspect these labels to determine which adapters
// are loaded in the pod.
Expand Down
18 changes: 6 additions & 12 deletions internal/modelcontroller/engine_vllm.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
package modelcontroller

import (
"context"
"sort"

kubeaiv1 "github.com/substratusai/kubeai/api/v1"
v1 "github.com/substratusai/kubeai/api/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
Expand All @@ -28,17 +26,17 @@ func (r *ModelReconciler) vLLMPodForModel(m *kubeaiv1.Model, c ModelConfig) *cor
"--model=" + vllmModelFlag,
"--served-model-name=" + m.Name,
}
if m.Spec.Adapters != nil {
args = append(args, "--enable-lora")
}
args = append(args, m.Spec.Args...)

env := []corev1.EnvVar{
{
env := []corev1.EnvVar{}

if m.Spec.Adapters != nil {
args = append(args, "--enable-lora")
env = append(env, corev1.EnvVar{
// https://docs.vllm.ai/en/latest/models/lora.html#dynamically-serving-lora-adapters
Name: "VLLM_ALLOW_RUNTIME_LORA_UPDATING",
Value: "True",
},
})
}

var envKeys []string
Expand Down Expand Up @@ -151,7 +149,3 @@ func (r *ModelReconciler) vLLMPodForModel(m *kubeaiv1.Model, c ModelConfig) *cor

return pod
}

func (r *ModelReconciler) vLLMLoadAdapter(ctx context.Context, pod *corev1.Pod, adapter v1.Adapter) error {
return nil
}

0 comments on commit ad72797

Please sign in to comment.