Envoy AI Gateway#

# 先给 helm/curl 也加代理
export http_proxy=http://218.16.121.13:1080
export https_proxy=http://218.16.121.13:1080
export HTTP_PROXY=http://218.16.121.13:1080
export HTTPS_PROXY=http://218.16.121.13:1080
export NO_PROXY=127.0.0.1,localhost,.svc,.cluster.local

# 1) 安装 Envoy Gateway（AI Gateway 官方要求先装这个，并带 AI Gateway values）
helm upgrade -i eg oci://docker.io/envoyproxy/gateway-helm \
  --version v0.0.0-latest \
  --namespace envoy-gateway-system \
  --create-namespace \
  -f https://raw.githubusercontent.com/envoyproxy/ai-gateway/main/manifests/envoy-gateway-values.yaml

kubectl wait --timeout=5m -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available

# 2) 安装 Envoy AI Gateway CRDs
helm upgrade -i aieg-crd oci://docker.io/envoyproxy/ai-gateway-crds-helm \
  --version v0.0.0-latest \
  --namespace envoy-ai-gateway-system \
  --create-namespace

# 3) 安装 Envoy AI Gateway controller
helm upgrade -i aieg oci://docker.io/envoyproxy/ai-gateway-helm \
  --version v0.0.0-latest \
  --namespace envoy-ai-gateway-system \
  --create-namespace

kubectl wait --timeout=5m -n envoy-ai-gateway-system deployment/ai-gateway-controller --for=condition=Available


cat << 'EOF' > mock-openai.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: llm-mock
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: mock-openai
  namespace: llm-mock
spec:
  replicas: 1
  selector:
    matchLabels:
      app: mock-openai
  template:
    metadata:
      labels:
        app: mock-openai
    spec:
      containers:
        - name: mock
          image: zerob13/mock-openai-api:latest
          ports:
            - containerPort: 3000
          env:
            - name: PORT
              value: "3000"
---
apiVersion: v1
kind: Service
metadata:
  name: mock-openai
  namespace: llm-mock
spec:
  selector:
    app: mock-openai
  ports:
    - name: http
      port: 3000
      targetPort: 3000
EOF

kubectl apply -f mock-openai.yaml





cat <<'EOF' > aieg-to-mock.yaml
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
  name: envoy-ai-gw
spec:
  controllerName: gateway.envoyproxy.io/gatewayclass-controller
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: envoy-ai-gw
  namespace: llm-mock
spec:
  gatewayClassName: envoy-ai-gw
  listeners:
    - name: http
      protocol: HTTP
      port: 80
---
apiVersion: gateway.envoyproxy.io/v1alpha1
kind: Backend
metadata:
  name: mock-openai-backend
  namespace: llm-mock
spec:
  endpoints:
    - fqdn:
        hostname: mock-openai.llm-mock.svc.cluster.local
        port: 3000
---
apiVersion: aigateway.envoyproxy.io/v1alpha1
kind: AIServiceBackend
metadata:
  name: mock-openai
  namespace: llm-mock
spec:
  schema:
    name: OpenAI
  backendRef:
    group: gateway.envoyproxy.io
    kind: Backend
    name: mock-openai-backend
---
apiVersion: aigateway.envoyproxy.io/v1alpha1
kind: AIGatewayRoute
metadata:
  name: mock-route
  namespace: llm-mock
spec:
  parentRefs:
    - group: gateway.networking.k8s.io
      kind: Gateway
      name: envoy-ai-gw
  rules:
    - matches:
        - headers:
            - type: Exact
              name: x-ai-eg-model
              value: mock-gpt
      backendRefs:
        # 省略 group/kind 时，默认指向同 namespace 的 AIServiceBackend
        - name: mock-openai
EOF

kubectl apply -f aieg-to-mock.yaml

cat << 'EOF' > envoy-ai-gateway-sglang.yaml
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
  name: envoy-ai-gateway-sglang
spec:
  controllerName: gateway.envoyproxy.io/gatewayclass-controller
---
apiVersion: gateway.envoyproxy.io/v1alpha1
kind: EnvoyProxy
metadata:
  name: envoy-ai-gateway-sglang
  namespace: llm-pd-demo
spec:
  provider:
    type: Kubernetes
    kubernetes:
      envoyDeployment:
        container:
          resources: {}
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: envoy-ai-gateway-sglang
  namespace: llm-pd-demo
spec:
  gatewayClassName: envoy-ai-gateway-sglang
  listeners:
    - name: http
      protocol: HTTP
      port: 80
  infrastructure:
    parametersRef:
      group: gateway.envoyproxy.io
      kind: EnvoyProxy
      name: envoy-ai-gateway-sglang
---
apiVersion: gateway.envoyproxy.io/v1alpha1
kind: ClientTrafficPolicy
metadata:
  name: sglang-client-buffer-limit
  namespace: llm-pd-demo
spec:
  targetRefs:
    - group: gateway.networking.k8s.io
      kind: Gateway
      name: envoy-ai-gateway-sglang
  connection:
    bufferLimit: 50Mi
---
apiVersion: gateway.envoyproxy.io/v1alpha1
kind: Backend
metadata:
  name: sglang-model-gateway-backend
  namespace: llm-pd-demo
spec:
  endpoints:
    - fqdn:
        hostname: sglang-model-gateway.llm-pd-demo.svc.cluster.local
        port: 30000
---
apiVersion: aigateway.envoyproxy.io/v1alpha1
kind: AIServiceBackend
metadata:
  name: sglang-openai-backend
  namespace: llm-pd-demo
spec:
  schema:
    name: OpenAI
  backendRef:
    group: gateway.envoyproxy.io
    kind: Backend
    name: sglang-model-gateway-backend
---
apiVersion: aigateway.envoyproxy.io/v1alpha1
kind: AIGatewayRoute
metadata:
  name: sglang-route
  namespace: llm-pd-demo
spec:
  parentRefs:
    - group: gateway.networking.k8s.io
      kind: Gateway
      name: envoy-ai-gateway-sglang
  rules:
    - matches:
        - headers:
            - type: Exact
              name: x-ai-eg-model
              value: qwen25-05b-pd
      backendRefs:
        - name: sglang-openai-backend
EOF

kubectl apply -f envoy-ai-gateway-sglang.yaml

# 导入到整个集群（server+agents）
k3d image import -c cluster-1 lmsysorg/sglang:v0.5.9-cu129-amd64

# 只导入到指定节点（可多次 --nodes）
k3d image import -c cluster-1 --nodes k3d-cluster-1-agent-0 lmsysorg/sglang:v0.5.9-cu129-amd64

# 导入多个镜像
docker pull lmsysorg/sglang:v0.5.9-cu129-amd64
docker pull envoyproxy/gateway-helm:v0.0.0-latest
docker pull envoyproxy/ai-gateway-crds-helm:v0.0.0-latest
docker pull envoyproxy/ai-gateway-helm:v0.0.0-latest
k3d image import -c cluster-1 lmsysorg/sglang:v0.5.9-cu129-amd64 envoyproxy/gateway-helm:v0.0.0-latest envoyproxy/ai-gateway-crds-helm:v0.0.0-latest envoyproxy/ai-gateway-helm:v0.0.0-latest