树莓派 / NanoPi / N100 低功耗集群：k3s 高可用、长安链、监控、GitOps 全栈落地

[上游光猫/路由] ──(WAN)──> [N100 网口1] (OpenWrt/iStoreOS 旁路网关)
[N100 网口2] ──> [PoE 交换机] ──> [Pi4-1] [Pi4-2] [N100 网口3(可选)]
所有设备同一 VLAN / 子网：192.168.10.0/24 (管理) + 10.10.10.0/24 (集群 Pod 网段)

# 启用 SSH、设置用户名/密码/hostname、配置 WiFi(可选)
# 烧录后首启自动完成 cloud-init

# BIOS 关闭 Secure Boot、开启 UEFI、关闭 CSM
# 分区：/boot/efi 1G + / 剩余 (ext4/btrfs/zfs)
# 安装 openssh-server、配置静态 IP

#!/usr/bin/env bash
# init-node.sh —— 以 root 运行
set -euo pipefail

HOSTNAME=$(hostname)
IP=$(ip -4 addr show scope global | awk '/inet/{print $2}' | cut -d/ -f1 | head -1)
echo ">>> 初始化 $HOSTNAME ($IP)"

# 1. 换源（中科大/阿里云）
sed -i 's|http://ports.ubuntu.com|https://mirrors.ustc.edu.cn|g' /etc/apt/sources.list.d/*.list /etc/apt/sources.list 2>/dev/null || true
apt update && apt upgrade -y

# 2. 基础工具
apt install -y curl wget vim htop iotop iftop net-tools ipvsadm conntrack \
    socat jq yq git ca-certificates gnupg lsb-release \
    nfs-common open-iscsi multipath-tools \
    linux-modules-extra-$(uname -r)  # 含 overlay、br_netfilter 等

# 3. 内核参数（K8s 必需）
cat >/etc/sysctl.d/99-k8s.conf <<'EOF'
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
net.ipv6.conf.all.forwarding        = 1
fs.inotify.max_user_watches         = 524288
fs.inotify.max_user_instances       = 512
vm.max_map_count                    = 262144
kernel.panic                        = 10
kernel.panic_on_oops                = 1
EOF
sysctl --system

# 4. 关闭 swap（K8s 要求）
swapoff -a
sed -i '/ swap / s/^/#/' /etc/fstab

# 5. containerd（k3s 自带，这里只配置镜像加速）
mkdir -p /etc/containerd
containerd config default | sed 's|sandbox_image = "registry.k8s.io/pause:3.6"|sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.9"|' > /etc/containerd/config.toml
sed -i 's|config_path = ""|config_path = "/etc/containerd/certs.d"|' /etc/containerd/config.toml
mkdir -p /etc/containerd/certs.d/docker.io
cat >/etc/containerd/certs.d/docker.io/hosts.toml <<'EOF'
server = "https://docker.io"
[host."https://docker.mirrors.ustc.edu.cn"]
  capabilities = ["pull", "resolve"]
EOF
systemctl restart containerd

# 6. 时间同步
timedatectl set-timezone Asia/Shanghai
systemctl enable --now systemd-timesyncd

# 7. 磁盘调度器（NVMe/SSD 用 none）
for d in /sys/block/*/queue/scheduler; do echo none > $d 2>/dev/null || true; done

echo ">>> $HOSTNAME 初始化完成，IP: $IP"

# 在所有控制平面节点跑（示例 VIP 192.168.10.100）
# 这里用 kube-vip（更云原生，无需额外 VRRP）

# N100 控制节点 (假设 IP 192.168.10.11)
export K3S_TOKEN="$(openssl rand -hex 32)"   # 记下来，其它控制节点要用
export K3S_CLUSTER_INIT="true"
curl -sfL https://rancher-mirror.rancher.cn/k3s/k3s-install.sh | \
  INSTALL_K3S_MIRROR=cn \
  INSTALL_K3S_VERSION="v1.28.5+k3s1" \
  INSTALL_K3S_EXEC="server \
    --cluster-init \
    --tls-san 192.168.10.100 \
    --node-ip 192.168.10.11 \
    --flannel-backend=wireguard-native \
    --disable=traefik \
    --disable=servicelb \
    --disable=local-storage \
    --write-kubeconfig-mode 644" \
  sh -s -

# 验证
kubectl get nodes -o wide

# Pi4-1 (192.168.10.12)、Pi4-2 (192.168.10.13)
export K3S_TOKEN="<第一节点生成的 TOKEN>"
curl -sfL https://rancher-mirror.rancher.cn/k3s/k3s-install.sh | \
  INSTALL_K3S_MIRROR=cn \
  INSTALL_K3S_VERSION="v1.28.5+k3s1" \
  INSTALL_K3S_EXEC="server \
    --server https://192.168.10.11:6443 \
    --node-ip 192.168.10.12 \
    --flannel-backend=wireguard-native \
    --disable=traefik \
    --disable=servicelb \
    --disable=local-storage \
    --write-kubeconfig-mode 644" \
  sh -s -

# 同理，去掉 --cluster-init 和 --tls-san
export K3S_TOKEN="<TOKEN>"
curl -sfL https://rancher-mirror.rancher.cn/k3s/k3s-install.sh | \
  INSTALL_K3S_MIRROR=cn \
  INSTALL_K3S_EXEC="agent \
    --server https://192.168.10.100:6443 \
    --node-ip 192.168.10.xx \
    --flannel-backend=wireguard-native" \
  sh -s -

# 在任意控制节点（需 kubectl 已配置）
kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip/main/manifests/rbac.yaml

# 创建 kube-vip 镜像别名（避免拉取超时）
kubectl apply -f - <<'EOF'
apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-vip
  namespace: kube-system
data:
  vip: "192.168.10.100"
  interface: "eth0"
  controlplane: "true"
  services: "true"
  leaderElection: "true"
  leaseDuration: "5"
  renewDeadline: "3"
  retryPeriod: "1"
EOF

kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip/main/manifests/daemonset.yaml

kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/v1.6.2/deploy/longhorn.yaml
# 等待所有 longhorn-* pod Running
# UI: kubectl -n longhorn-system port-forward svc/longhorn-frontend 8080:80
# 设置：默认副本数 3、过度配比 200%、备份目标 S3/NFS/MinIO

helm repo add cilium https://helm.cilium.io --force-update
helm install cilium cilium/cilium --version 1.15.5 \
  --namespace kube-system \
  --set kubeProxyReplacement=true \
  --set ipam.mode=kubernetes \
  --set ipv4NativeRoutingCIDR=10.10.10.0/24 \
  --set hubble.enabled=true,hubble.ui.enabled=true \
  --set prometheus.enabled=true,operator.prometheus.enabled=true

# external-dns 同步 Service/Ingress 到 Cloudflare
helm repo add external-dns https://kubernetes-sigs.github.io/external-dns/
helm install external-dns external-dns/external-dns --version 1.14.0 \
  --namespace kube-system \
  --set provider=cloudflare \
  --set env[0].name=CF_API_TOKEN,env[0].value=xxx \
  --set policy=sync \
  --set txtOwnerId=my-cluster

# k3s 自带 Traefik，已禁用；改用 Nginx Ingress Controller
helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
helm install ingress-nginx ingress-nginx/ingress-nginx --version 4.10.1 \
  --namespace ingress-nginx --create-namespace \
  --set controller.service.type=LoadBalancer \
  --set controller.service.externalIPs={192.168.10.100} \
  --set controller.publishService.enabled=true \
  --set controller.metrics.enabled=true

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm install monitoring prometheus-community/kube-prometheus-stack --version 56.0.0 \
  --namespace monitoring --create-namespace \
  --set prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.storageClassName=longhorn \
  --set prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage=50Gi \
  --set grafana.persistence.enabled=true,grafana.persistence.storageClassName=longhorn,grafana.persistence.size=10Gi \
  --set alertmanager.alertmanagerSpec.storage.volumeClaimTemplate.spec.storageClassName=longhorn \
  --set alertmanager.alertmanagerSpec.storage.volumeClaimTemplate.spec.resources.requests.storage=10Gi

kubectl create namespace argocd
kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/v2.9.3/manifests/install.yaml
# 修改 argocd-server 为 LoadBalancer 或 Ingress
kubectl patch svc argocd-server -n argocd -p '{"spec":{"type":"LoadBalancer","externalIPs":["192.168.10.100"]}}'
# 初始密码
kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d

# 1. 镜像打包
ctr -n k8s.io images export /tmp/k3s-images.tar \
  $(ctr -n k8s.io images ls -q | grep -E 'rancher|cilium|longhorn|prometheus|grafana|argocd|traefik|nginx|external-dns|kube-vip|pause|coredns|metrics-server|local-path-provisioner')

# 2. Helm Chart 打包
mkdir -p /tmp/charts
for chart in cilium longhorn prometheus-community/kube-prometheus-stack ingress-nginx/ingress-nginx argoproj/argo-cd external-dns/external-dns; do
  helm pull $chart --untar -d /tmp/charts --version latest
done

# 3. 打包发运
tar -czf k3s-airgap-$(date +%Y%m%d).tar.gz /tmp/k3s-images.tar /tmp/charts /usr/local/bin/init-node.sh
# 目标机器：解压 → ctr images import → helm install -f values.yaml ./charts/xxx

# ansible/inventory.yml
all:
  vars:
    ansible_user: ubuntu
    ansible_become: yes
    k3s_version: "v1.28.5+k3s1"
    cluster_vip: "192.168.10.100"
  children:
    control:
      hosts:
        n100: { ansible_host: 192.168.10.11, node_ip: 192.168.10.11 }
        pi4-1: { ansible_host: 192.168.10.12, node_ip: 192.168.10.12 }
        pi4-2: { ansible_host: 192.168.10.13, node_ip: 192.168.10.13 }
    worker:
      hosts: {}

# 一键全生命周期
ansible-playbook -i inventory.yml site.yml --tags bootstrap   # 首次部署
ansible-playbook -i inventory.yml site.yml --tags upgrade     # 版本升级
ansible-playbook -i inventory.yml site.yml --tags rotate-cert # 证书轮换
ansible-playbook -i inventory.yml site.yml --tags backup      # etcd/Longhorn 备份
ansible-playbook -i inventory.yml site.yml --tags scale       # 扩缩容节点

# cluster-bootstrap.sh —— 在第一控制节点以 root 运行
#!/usr/bin/env bash
set -euo pipefail
source /etc/os-release
echo ">>> Ubuntu $VERSION_ID 集群引导开始"

# 1. 所有节点跑 init-node.sh（通过 Ansible 或 pdsh 并行）
# ansible all -m script -a ./init-node.sh

# 2. 部署 k3s HA（按上文 3.2-3.5）
# ansible-playbook -i inventory.yml k3s-ha.yml

# 3. 部署核心插件（Helm 统一管理）
helm repo add longhorn https://charts.longhorn.io
helm repo add cilium https://helm.cilium.io
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
helm repo add argo https://argoproj.github.io/argo-helm
helm repo update

# Longhorn
helm upgrade --install longhorn longhorn/longhorn -n longhorn-system --create-namespace \
  --version 1.6.2 --set defaultSettings.defaultReplicaCount=3

# Cilium
helm upgrade --install cilium cilium/cilium -n kube-system --version 1.15.5 \
  --set kubeProxyReplacement=true --set ipam.mode=kubernetes \
  --set hubble.enabled=true,hubble.ui.enabled=true

# Monitoring
helm upgrade --install monitoring prometheus-community/kube-prometheus-stack -n monitoring --create-namespace \
  --version 56.0.0 --set prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.storageClassName=longhorn

# ArgoCD
helm upgrade --install argocd argo/argo-cd -n argocd --create-namespace \
  --version 7.6.0 --set server.service.type=LoadBalancer,server.service.externalIPs={192.168.10.100}

echo ">>> 集群引导完成！访问："
echo "  Longhorn UI:     https://longhorn.${DOMAIN}"
echo "  Grafana:         https://grafana.${DOMAIN}"
echo "  ArgoCD:          https://argocd.${DOMAIN}"
echo "  kube-vip VIP:    https://192.168.10.100:6443"