k3d#

rancher/k3d Github stars Github forks Language Last Tag Last commit

常用命令#

k3d cluster list
kubectl config get-contexts

k3d kubeconfig get cluster-1 > ~/.kube/cluster-1
chmod 600 ~/.kube/cluster-1
k3d kubeconfig get cluster-2 > ~/.kube/cluster-2
chmod 600 ~/.kube/cluster-2
k3d kubeconfig get cluster-3 > ~/.kube/cluster-3
chmod 600 ~/.kube/cluster-3
k3d kubeconfig get cluster-4 > ~/.kube/cluster-4
chmod 600 ~/.kube/cluster-4
k3d kubeconfig get cluster-5 > ~/.kube/cluster-5
chmod 600 ~/.kube/cluster-5
k3d kubeconfig get cluster-6 > ~/.kube/cluster-6
chmod 600 ~/.kube/cluster-6
k3d kubeconfig get cluster-7 > ~/.kube/cluster-7
chmod 600 ~/.kube/cluster-7
k3d kubeconfig get cluster-8 > ~/.kube/cluster-8
chmod 600 ~/.kube/cluster-8
k3d kubeconfig get cluster-9 > ~/.kube/cluster-9
chmod 600 ~/.kube/cluster-9


cat <<'EOF' >> ~/.bashrc

alias kctx='kubectl config use-context'
alias c1='kctx k3d-cluster-1'
alias c2='kctx k3d-cluster-2'
alias c3='kctx k3d-cluster-3'
alias c4='kctx k3d-cluster-4'
alias k1='KUBECONFIG=~/.kube/cluster-1 kubectl'
alias k2='KUBECONFIG=~/.kube/cluster-2 kubectl'
alias k3='KUBECONFIG=~/.kube/cluster-3 kubectl'
alias k4='KUBECONFIG=~/.kube/cluster-4 kubectl'

EOF

安装#

# 安装 k3d
wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash

# https://hub.docker.com/r/rancher/k3s/tags

cat << 'EOF' > k3d-config-cluster-1.yaml
apiVersion: k3d.io/v1alpha5
kind: Simple
metadata:
  name: cluster-1
image: rancher/k3s:v1.35.1-k3s1
servers: 1
agents: 2

kubeAPI:
  host: "127.0.0.1"
  hostIP: "0.0.0.0"
  hostPort: "8521"
registries:
  config: |
    mirrors:
      "docker.io":
        endpoint:
          - https://hub.kingye.me
EOF

k3d cluster create --config k3d-config-cluster-1.yaml --verbose

安装配置#

# 跑多个 k3s,要提高 inotify + file limit
sudo tee /etc/sysctl.d/99-k3d-inotify.conf >/dev/null <<'EOF'
fs.inotify.max_user_instances = 4048
fs.inotify.max_user_watches = 1048576
fs.inotify.max_queued_events = 65536
fs.file-max = 2097152
EOF

sudo sysctl --system

GPU#

k3s-gpu#

# 为 Docker daemon 配置代理(永久生效)
sudo mkdir -p /etc/systemd/system/docker.service.d
sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf > /dev/null <<'EOF'
[Service]
Environment="HTTP_PROXY=http://218.16.121.13:1080"
Environment="HTTPS_PROXY=http://218.16.121.13:1080"
Environment="NO_PROXY=localhost,127.0.0.1"
EOF

sudo systemctl daemon-reload
sudo systemctl restart docker


# https://hub.docker.com/r/nvidia/cuda/tags

cat << 'DOCKERFILE_EOF' > Dockerfile
FROM nvidia/cuda:12.8.1-base-ubuntu24.04

ARG DEBIAN_FRONTEND=noninteractive

# 版本参数
ARG K3S_VERSION="v1.35.1+k3s1"
ARG KUBECTL_VERSION="v1.35.1"
ARG CRICTL_VERSION="v1.35.0"

# 下载代理前缀(可为空)
# 例如:
#   "https://proxy.kingye.me/proxy/"
# 或者置空 "" 走直连
ARG DOWNLOAD_PROXY_PREFIX="https://proxy.kingye.me/proxy/"

SHELL ["/bin/bash", "-o", "pipefail", "-c"]

ENV PATH="/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"

# 1) 基础工具 + containerd(带 ctr) + CNI plugins + NVIDIA Container Toolkit + fuse-overlayfs
RUN set -eux; \
    apt-get update; \
    apt-get install -y --no-install-recommends \
      ca-certificates curl gnupg tar \
      iptables iproute2 \
      iputils-ping conntrack ethtool \
      containernetworking-plugins \
      fuse-overlayfs \
      containerd; \
    \
    # NVIDIA Container Toolkit repo
    mkdir -p /usr/share/keyrings; \
    curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
      | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg; \
    curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
      | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \
      > /etc/apt/sources.list.d/nvidia-container-toolkit.list; \
    apt-get update; \
    apt-get install -y --no-install-recommends \
      nvidia-container-toolkit \
      nvidia-container-runtime \
      libnvidia-container-tools; \
    \
    # 把发行版 CNI 插件软链到 k3s 常用路径
    mkdir -p /opt/cni/bin; \
    for d in /usr/lib/cni /usr/libexec/cni; do \
      if [ -d "$d" ]; then \
        find "$d" -maxdepth 1 -type f -exec ln -sf {} /opt/cni/bin/ \; ; \
      fi; \
    done; \
    \
    # 基础 sanity checks(build 阶段没有 daemon,失败允许)
    command -v ctr; \
    ctr version || true; \
    command -v nvidia-container-runtime; \
    nvidia-container-runtime --version || true; \
    \
    rm -rf /var/lib/apt/lists/*

# 2) 安装 k3s / kubectl / crictl(二进制下载,统一重试逻辑)+ crictl 默认配置
RUN set -eux; \
    dl() { \
      local url="$1"; \
      local out="$2"; \
      curl -fL --retry 5 --retry-delay 1 --retry-connrefused \
        -o "$out" "${DOWNLOAD_PROXY_PREFIX}${url}"; \
    }; \
    \
    # k3s
    dl "https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}/k3s" /bin/k3s; \
    chmod +x /bin/k3s; \
    /bin/k3s --version; \
    \
    # kubectl
    dl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" /bin/kubectl; \
    chmod +x /bin/kubectl; \
    /bin/kubectl version --client=true --output=yaml; \
    \
    # crictl(避免 Ubuntu 24.04 上 cri-tools 包问题)
    dl "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" /tmp/crictl.tgz; \
    tar -C /usr/local/bin -xzf /tmp/crictl.tgz crictl; \
    rm -f /tmp/crictl.tgz; \
    command -v crictl; \
    /usr/local/bin/crictl --version; \
    \
    # crictl endpoint 固化(稳定,不依赖 deprecated 自动探测)
    mkdir -p /etc; \
    cat > /etc/crictl.yaml <<'EOF'
runtime-endpoint: unix:///run/k3s/containerd/containerd.sock
image-endpoint: unix:///run/k3s/containerd/containerd.sock
timeout: 10
debug: false
EOF

RUN set -eux; \
    cat > /usr/local/bin/ctr <<'EOF'
#!/usr/bin/env bash
set -euo pipefail

REAL_CTR="/usr/bin/ctr"
K3S_SOCK="/run/k3s/containerd/containerd.sock"
DEFAULT_NS="k8s.io"

has_addr=0
has_ns=0

for ((i=1; i<=$#; i++)); do
  arg="${!i}"
  case "$arg" in
    -a|--address)
      has_addr=1
      ;;
    -n|--namespace)
      has_ns=1
      ;;
    --address=* )
      has_addr=1
      ;;
    --namespace=* )
      has_ns=1
      ;;
  esac
done

args=()

# 默认指向 k3s 内嵌 containerd(除非用户自己指定了 -a/--address)
if [[ $has_addr -eq 0 && -S "$K3S_SOCK" ]]; then
  args+=(-a "$K3S_SOCK")
fi

# 默认 namespace = k8s.io(除非用户自己指定了 -n/--namespace)
if [[ $has_ns -eq 0 ]]; then
  args+=(-n "$DEFAULT_NS")
fi

exec "$REAL_CTR" "${args[@]}" "$@"
EOF

RUN chmod +x /usr/local/bin/ctr; \
    chmod 0644 /etc/crictl.yaml

# 3) 写 entrypoint(运行时创建 /run 下的兼容软链接)
RUN set -eux; \
    cat > /usr/local/bin/entrypoint.sh <<'ENTRYPOINT_EOF'
#!/usr/bin/env bash
set -euo pipefail

# ---- runtime fixes for tooling ----
# /run 是运行时目录;这里创建兼容路径和软链接,方便 ctr/nerdctl 默认地址使用
mkdir -p /run/containerd /run/k3s/containerd 2>/dev/null || true
ln -sf /run/k3s/containerd/containerd.sock /run/containerd/containerd.sock || true
# -----------------------------------

# 无参数:默认跑 k3s server
if [[ $# -eq 0 ]]; then
  exec /bin/k3s server --disable=traefik --disable=servicelb
fi

case "$1" in
  server|agent)
    exec /bin/k3s "$@"
    ;;
  k3s)
    shift
    exec /bin/k3s "$@"
    ;;
  kubectl)
    shift
    exec /bin/kubectl "$@"
    ;;
  ctr|crictl)
    exec "$@"
    ;;
  --help|-h)
    exec /bin/k3s --help
    ;;
  *)
    # 调试入口:bash / sh / nvidia-smi / cat ...
    exec "$@"
    ;;
esac
ENTRYPOINT_EOF

RUN chmod +x /usr/local/bin/entrypoint.sh

ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
CMD ["server", "--disable=traefik", "--disable=servicelb"]
DOCKERFILE_EOF

# 构建与推送(建议带 --pull,确保基底镜像更新)
docker build --pull -t ikingye/k3s-gpu:v1.35.1-k3s1-cuda12.8.1 .
docker push ikingye/k3s-gpu:v1.35.1-k3s1-cuda12.8.1

# 依赖
sudo apt-get update
sudo apt-get install -y curl ca-certificates gnupg

# 添加仓库 key
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
  | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg

# 添加 repo 列表
curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
  | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \
  | sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list > /dev/null

sudo apt-get update
sudo apt-get install -y nvidia-container-toolkit

sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker

# 验证 GPU 容器
docker run --rm --gpus all nvidia/cuda:12.8.1-base-ubuntu24.04 nvidia-smi

docker run --rm --gpus all ikingye/k3s-gpu:v1.35.1-k3s1-cuda13.1.0 nvidia-smi

docker run --rm --gpus all rancher/k3s:v1.35.1-k3s1 nvidia-smi

docker run --rm --gpus all ikingye/k3s-gpu:v1.35.1-k3s1-cuda12.8.1 nvidia-smi


docker exec -it k3d-cluster-1-agent-0 ctr -n k8s.io images ls
docker exec -it k3d-cluster-1-agent-0 crictl images

# 先在宿主机 pull 下来
docker pull lmsysorg/sglang:v0.5.9-cu129-amd64
docker pull envoyproxy/gateway-helm:v0.0.0-latest
docker pull envoyproxy/ai-gateway-crds-helm:v0.0.0-latest
docker pull envoyproxy/ai-gateway-helm:v0.0.0-latest

# 导入到整个集群(server+agents)
k3d image import -c cluster-1 lmsysorg/sglang:v0.5.9-cu129-amd64

# 只导入到指定节点(可多次 --nodes)
k3d image import -c cluster-1 --nodes k3d-cluster-1-agent-0 lmsysorg/sglang:v0.5.9-cu129-amd64

# 导入多个镜像
k3d image import -c cluster-1 lmsysorg/sglang:v0.5.9-cu129-amd64 envoyproxy/gateway-helm:v0.0.0-latest envoyproxy/ai-gateway-crds-helm:v0.0.0-latest envoyproxy/ai-gateway-helm:v0.0.0-latest

k3d image import -c cluster-3 ccr-241y349u-pub.cnc.bj.baidubce.com/ecs-public/sglang:4090-0.5.8-cu126


cat << 'CREATE_EOF' > create_k3s.sh
#!/usr/bin/env bash
set -euo pipefail

usage() {
  cat <<'USAGE'
Usage:
  bash create_k3s.sh <start> <end> [options]
  bash create_k3s.sh --range A:B [options]
  bash create_k3s.sh --start A --end B [options]

Options:
  --start N                 Start cluster index (e.g. 1)
  --end N                   End cluster index (e.g. 4)
  --range A:B               Shortcut for --start A --end B

  --image IMAGE             k3s image
  --servers N               servers count (default: 1)
  --agents N                agents count (default: 2)

  --hostport-base N         kubeAPI hostPort = base + i (default: 8520)
  --state-root PATH         per-node k3s state root on host (default: /mnt/data1/k3d-data)

  --data HOST:CONTAINER     bind mount into ALL nodes; repeatable
                            e.g. --data=/mnt/data1:/mnt/data1 --data=/mnt/data2:/mnt/data2

  --tls-san IP              repeatable, applied to servers (default: 124.237.232.51 + 172.16.16.63)
  --device-plugin-url URL   nvidia device plugin manifest URL
  --no-preflight            skip docker run nvidia-smi preflight
  -h|--help                 show help

Examples:
  bash create_k3s.sh 1 4 --state-root /mnt/data1/k3d-data \
    --data=/mnt/data1:/mnt/data1 --data=/mnt/data2:/mnt/data2

  bash create_k3s.sh --range 5:5 --agents 3 --servers 1 \
    --state-root /mnt/nvme/k3d-data \
    --data=/mnt/nvme:/mnt/nvme
USAGE
}

is_int() { [[ "${1:-}" =~ ^[0-9]+$ ]]; }

# ---------------- defaults ----------------
start=""
end=""

IMAGE="${IMAGE:-ikingye/k3s-gpu:v1.35.1-k3s1-cuda12.6.3}"
SERVERS="${SERVERS:-1}"
AGENTS="${AGENTS:-2}"

HOSTPORT_BASE="${HOSTPORT_BASE:-8520}"
STATE_ROOT="${STATE_ROOT:-/mnt/data1/k3d-data}"

DEVICE_PLUGIN_URL="${DEVICE_PLUGIN_URL:-https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.18.2/deployments/static/nvidia-device-plugin.yml}"

# repeatable args
DATA_MOUNTS=()         # host:container
TLS_SANS=()            # IPs
NO_PREFLIGHT=0

# default TLS sans (if user doesn't pass any --tls-san)
TLS_SANS_DEFAULT=("124.237.232.51" "172.16.16.63")

# ---------------- arg parsing ----------------
positional=()

while [[ $# -gt 0 ]]; do
  case "$1" in
    --start)
      start="${2:-}"; shift 2;;
    --end)
      end="${2:-}"; shift 2;;
    --range)
      r="${2:-}"; shift 2
      start="${r%%:*}"
      end="${r##*:}"
      ;;
    --range=*)
      r="${1#*=}"; shift
      start="${r%%:*}"
      end="${r##*:}"
      ;;
    --image)
      IMAGE="${2:-}"; shift 2;;
    --image=*)
      IMAGE="${1#*=}"; shift;;
    --servers)
      SERVERS="${2:-}"; shift 2;;
    --servers=*)
      SERVERS="${1#*=}"; shift;;
    --agents)
      AGENTS="${2:-}"; shift 2;;
    --agents=*)
      AGENTS="${1#*=}"; shift;;

    --hostport-base)
      HOSTPORT_BASE="${2:-}"; shift 2;;
    --hostport-base=*)
      HOSTPORT_BASE="${1#*=}"; shift;;

    --state-root)
      STATE_ROOT="${2:-}"; shift 2;;
    --state-root=*)
      STATE_ROOT="${1#*=}"; shift;;

    --data)
      DATA_MOUNTS+=("${2:-}"); shift 2;;
    --data=*)
      DATA_MOUNTS+=("${1#*=}"); shift;;

    --tls-san)
      TLS_SANS+=("${2:-}"); shift 2;;
    --tls-san=*)
      TLS_SANS+=("${1#*=}"); shift;;

    --device-plugin-url)
      DEVICE_PLUGIN_URL="${2:-}"; shift 2;;
    --device-plugin-url=*)
      DEVICE_PLUGIN_URL="${1#*=}"; shift;;

    --no-preflight)
      NO_PREFLIGHT=1; shift;;

    -h|--help)
      usage; exit 0;;

    --)
      shift; break;;

    -*)
      echo "Unknown option: $1" >&2
      usage; exit 1;;

    *)
      positional+=("$1"); shift;;
  esac
done

# positional start/end fallback
if [[ -z "${start}" && ${#positional[@]} -ge 1 ]]; then start="${positional[0]}"; fi
if [[ -z "${end}"   && ${#positional[@]} -ge 2 ]]; then end="${positional[1]}"; fi

# validations
if [[ -z "${start}" || -z "${end}" ]]; then
  echo "ERROR: start/end required." >&2
  usage; exit 1
fi
if ! is_int "${start}" || ! is_int "${end}" || ! is_int "${SERVERS}" || ! is_int "${AGENTS}" || ! is_int "${HOSTPORT_BASE}"; then
  echo "ERROR: start/end/servers/agents/hostport-base must be integers." >&2
  exit 1
fi
if (( start > end )); then
  echo "ERROR: start(${start}) > end(${end})." >&2
  exit 1
fi
if (( SERVERS < 1 || AGENTS < 0 )); then
  echo "ERROR: servers must be >=1 and agents must be >=0." >&2
  exit 1
fi

# default TLS sans if none provided
if [[ ${#TLS_SANS[@]} -eq 0 ]]; then
  TLS_SANS=("${TLS_SANS_DEFAULT[@]}")
fi

preflight() {
  if (( NO_PREFLIGHT == 1 )); then
    echo "==> Preflight skipped (--no-preflight)"
    return 0
  fi
  echo "==> Preflight GPU: docker run nvidia-smi"
  docker run --rm --gpus all "${IMAGE}" nvidia-smi >/dev/null
}

gen_k3d_config() {
  local i="$1"
  local cfg="k3d-config-cluster-${i}.yaml"
  local hostPort="$((HOSTPORT_BASE + i))"

  cat > "${cfg}" <<EOF
apiVersion: k3d.io/v1alpha5
kind: Simple
metadata:
  name: cluster-${i}

image: ${IMAGE}

servers: ${SERVERS}
agents: ${AGENTS}

kubeAPI:
  host: "127.0.0.1"
  hostIP: "0.0.0.0"
  hostPort: "${hostPort}"

# 关键:把每个节点的 /var/lib/rancher/k3s 单独挂到 state-root
volumes:
EOF

  # per-server state
  for ((s=0; s<SERVERS; s++)); do
    cat >> "${cfg}" <<EOF
  - volume: ${STATE_ROOT}/cluster-${i}/server-${s}:/var/lib/rancher/k3s
    nodeFilters:
      - server:${s}
EOF
  done

  # per-agent state
  for ((a=0; a<AGENTS; a++)); do
    cat >> "${cfg}" <<EOF
  - volume: ${STATE_ROOT}/cluster-${i}/agent-${a}:/var/lib/rancher/k3s
    nodeFilters:
      - agent:${a}
EOF
  done

  # shared mounts from --data
  for m in "${DATA_MOUNTS[@]}"; do
    cat >> "${cfg}" <<EOF
  - volume: ${m}
    nodeFilters:
      - server:*
      - agent:*
EOF
  done

  cat >> "${cfg}" <<'EOF'
options:
  runtime:
    gpuRequest: all
  k3s:
    extraArgs:
      # 避免每个 Pod 都写 runtimeClassName
      - arg: "--default-runtime=nvidia"
        nodeFilters:
          - agent:*
      - arg: "--node-taint=node-role.kubernetes.io/control-plane=true:NoSchedule"
        nodeFilters:
          - server:*
      - arg: "--disable=traefik"
        nodeFilters:
          - server:*
      - arg: "--disable=servicelb"
        nodeFilters:
          - server:*
      - arg: "--kubelet-arg=image-gc-high-threshold=99"
        nodeFilters:
          - server:*
          - agent:*
      - arg: "--kubelet-arg=image-gc-low-threshold=95"
        nodeFilters:
          - server:*
          - agent:*
      - arg: "--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%"
        nodeFilters:
          - server:*
          - agent:*
EOF

  # tls sans (repeatable)
  for ip in "${TLS_SANS[@]}"; do
    cat >> "${cfg}" <<EOF
      - arg: --tls-san=${ip}
        nodeFilters:
          - server:*
EOF
  done

  cat >> "${cfg}" <<'EOF'

  kubeconfig:
    updateDefaultKubeconfig: true
    switchCurrentContext: true

registries:
  config: |
    mirrors:
      "docker.io":
        endpoint:
          - "https://hub.kingye.me"
      "ghcr.io":
        endpoint:
          - "https://ghcr.kingye.me"
      "gcr.io":
        endpoint:
          - "https://gcr.kingye.me"
      "k8s.gcr.io":
        endpoint:
          - "https://k8sgcr.kingye.me"
      "registry.k8s.io":
        endpoint:
          - "https://k8s.kingye.me"
      "quay.io":
        endpoint:
          - "https://quay.kingye.me"
      "mcr.microsoft.com":
        endpoint:
          - "https://mcr.kingye.me"
      "docker.elastic.co":
        endpoint:
          - "https://elastic.kingye.me"
      "nvcr.io":
        endpoint:
          - "https://nvcr.kingye.me"
EOF
}

ensure_dirs() {
  local i="$1"
  local base="${STATE_ROOT}/cluster-${i}"

  mkdir -p "${base}"

  for ((s=0; s<SERVERS; s++)); do
    mkdir -p "${base}/server-${s}"
  done
  for ((a=0; a<AGENTS; a++)); do
    mkdir -p "${base}/agent-${a}"
  done

  chmod 755 "${base}" || true
  for p in "${base}"/*; do
    [[ -e "$p" ]] && chmod 755 "$p" || true
  done
}

wipe_state() {
  local i="$1"
  local base="${STATE_ROOT}/cluster-${i}"

  for ((s=0; s<SERVERS; s++)); do
    [[ -d "${base}/server-${s}" ]] && find "${base}/server-${s}" -mindepth 1 -maxdepth 1 -exec rm -rf {} + || true
  done
  for ((a=0; a<AGENTS; a++)); do
    [[ -d "${base}/agent-${a}" ]] && find "${base}/agent-${a}" -mindepth 1 -maxdepth 1 -exec rm -rf {} + || true
  done
}

install_device_plugin() {
  kubectl apply -f "${DEVICE_PLUGIN_URL}"

  # 幂等 patch:runtimeClassName + nodeSelector
  kubectl patch ds -n kube-system nvidia-device-plugin-daemonset --type='merge' -p '{
    "spec": {
      "template": {
        "spec": {
          "runtimeClassName": "nvidia",
          "nodeSelector": { "accelerator": "nvidia" }
        }
      }
    }
  }' || true

  kubectl set env -n kube-system ds/nvidia-device-plugin-daemonset FAIL_ON_INIT_ERROR=false
  kubectl rollout status -n kube-system ds/nvidia-device-plugin-daemonset --timeout=180s

  kubectl get pods -n kube-system -l name=nvidia-device-plugin-ds -o wide || true
}

create_cluster() {
  local i="$1"

  echo
  echo "=============================="
  echo "==> cluster-${i} (servers=${SERVERS}, agents=${AGENTS})"
  echo "    state-root=${STATE_ROOT}"
  echo "    data-mounts=${#DATA_MOUNTS[@]}"
  echo "=============================="

  gen_k3d_config "${i}"
  ensure_dirs "${i}"

  k3d cluster delete "cluster-${i}" || true
  wipe_state "${i}"
  docker network rm "k3d-cluster-${i}" 2>/dev/null || true

  k3d cluster create --config "k3d-config-cluster-${i}.yaml" --verbose

  kubectl config use-context "k3d-cluster-${i}"

  kubectl get nodes -o wide
  kubectl get runtimeclass || true

  # 给所有 agent 打标签
  for ((a=0; a<AGENTS; a++)); do
    kubectl label node "k3d-cluster-${i}-agent-${a}" accelerator=nvidia --overwrite
  done

  install_device_plugin

  kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\tallocatable="}{.status.allocatable.nvidia\.com/gpu}{"\tcapacity="}{.status.capacity.nvidia\.com/gpu}{"\n"}{end}' || true
}

main() {
  preflight
  for ((i=start; i<=end; i++)); do
    create_cluster "${i}"
  done
}

main "$@"
CREATE_EOF

bash create_k3s.sh

bash create_k3s.sh --range 5:5 --servers 1 --agents 2 \
--state-root /home/k3d-data --data=/home:/home \
--tls-san 1.180.13.251 --tls-san 1.180.13.251 172.16.48.74