EKS Cluster <> ASG 연동 (cluster-autoscaler)

helm 과 yaml 형식으로 있음 yaml 을 선호

namespace별로 alb 생성 됨

--- --- --- --- --- --- --- --- --- --- --- --- --- --- ---

apiVersion: v1
kind: ServiceAccount
metadata:
  name: cluster-autoscaler
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: cluster-autoscaler
rules:
  - apiGroups: [""]
    resources: ["events","endpoints"]
    verbs: ["create","patch"]
  - apiGroups: [""]
    resources: ["pods/eviction"]
    verbs: ["create"]
  - apiGroups: [""]
    resources: ["pods/status"]
    verbs: ["update"]
  - apiGroups: [""]
    resources: ["endpoints"]
    resourceNames: ["cluster-autoscaler"]
    verbs: ["get","update"]
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["watch","list","get","update"]
  - apiGroups: [""]
    resources: ["pods","services","replicationcontrollers","persistentvolumeclaims","persistentvolumes"]
    verbs: ["watch","list","get"]
  - apiGroups: ["extensions"]
    resources: ["replicasets","daemonsets"]
    verbs: ["watch","list","get"]
  - apiGroups: ["policy"]
    resources: ["poddisruptionbudgets"]
    verbs: ["watch","list"]
  - apiGroups: ["apps"]
    resources: ["statefulsets","replicasets","daemonsets"]
    verbs: ["watch","list","get"]
  - apiGroups: ["coordination.k8s.io"]
    resources: ["leases"]
    verbs: ["get","watch","list","update","create","patch"]
  - apiGroups: ["storage.k8s.io"]
    resources:
      - csistoragecapacities
      - csinodes
      - csidrivers
      - storageclasses
    verbs: ["list", "watch"]
  - apiGroups: ["batch"]
    resources: ["jobs"]
    verbs: ["list", "watch"]
  - apiGroups: [""]
    resources: ["namespaces"]
    verbs: ["list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: cluster-autoscaler
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-autoscaler
subjects:
- kind: ServiceAccount
  name: cluster-autoscaler
  namespace: kube-system
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cluster-autoscaler
  namespace: kube-system
  labels:
    app: cluster-autoscaler
spec:
  replicas: 1
  selector:
    matchLabels:
      app: cluster-autoscaler
  template:
    metadata:
      labels:
        app: cluster-autoscaler
      annotations:
        cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
    spec:
      serviceAccountName: cluster-autoscaler
      containers:
      - name: cluster-autoscaler
        image: k8s.gcr.io/autoscaling/cluster-autoscaler:v1.25.0
        command:
          - ./cluster-autoscaler
          - --v=4
          - --stderrthreshold=info
          - --cloud-provider=aws
          - --skip-nodes-with-local-storage=false
          - --expander=least-waste
          - --balance-similar-node-groups
          - --nodes=2:3:dev-sw-eks-asg          # 실제 ASG 이름과 min/max 노드 수로 수정
          - --scale-down-delay-after-add=1m
          - --ignore-daemonsets-utilization=true
          - --scan-interval=10s
        resources:
          requests:
            cpu: 100m
            memory: 300Mi
          limits:
            cpu: 100m
            memory: 300Mi
      tolerations:
      - key: "node-role.kubernetes.io/control-plane"
        effect: NoSchedule
      - key: "node-role.kubernetes.io/master"
        effect: NoSchedule
---

k apply -f cluster-autoscaler.yaml

--------------------------------------------------------------------------------------------------------------------------------------

helm으로 설치
--------------------------------------------------------------------------------------------------------------------------------------

helm 에 cluster-autoscaler 사용 

별로 생성한 asg 연동 하여 증설 감축 

 

--helm 설치 
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
helm repo add autoscaler https://kubernetes.github.io/autoscaler
helm repo update

 

-- cluster-autoscaler 설치

helm install cluster-autoscaler autoscaler/cluster-autoscaler \
  --namespace kube-system \
  --create-namespace \
  --set autoscalingGroups[0].name=dev-sw-eks-asg \  #asg 네임으로만 지정해 줄 때 // 직접 지정 해줘야 컨트롤 가능함
  --set autoscalingGroups[0].maxSize=5 \
  --set autoscalingGroups[0].minSize=2 \

  --set autoDiscovery.tags=k8s.io/cluster-autoscaler/enabled=true \
  --set awsRegion=ap-northeast-2 \
  --set rbac.serviceAccount.create=true \
  --set rbac.serviceAccount.name=cluster-autoscaler \
  --set extraArgs.balance-similar-node-groups=true \
  --set extraArgs.skip-nodes-with-local-storage=false \
  --set extraArgs.expander=least-waste \
  --set extraArgs.ignore-daemonsets-utilization=true \
  --set extraArgs.scan-interval=10s \
  --set resources.requests.cpu=100m \
  --set resources.requests.memory=300Mi \
  --set resources.limits.cpu=100m \
  --set resources.limits.memory=300Mi \

  --set extraArgs.scale-down-delay-after-add=1m \
  --set extraArgs.scale-down-delay-after-delete=30s \
  --set extraArgs.scale-down-delay-after-failure=2m

-------------------------------------------------------------------------------------------------------

  --set autoDiscovery.clusterName=dev-sw-eks-cluster \ #노드 그룹 사용 할 때  asg 와 둘 중 하나만 사용 가능

-- Cluster Autoscaler 정상 연결확인 로그 
kubectl -n kube-system logs -l app.kubernetes.io/name=aws-cluster-autoscaler -f 
*Registering ASG dev-sw-eks-asg 부분이 있어야 함 
*10250 포트 열려 있어야 함 kubelet api 포트

 

-- 테스트 

test.yaml  생성 하여 파드 추가 

apiVersion: v1
kind: Pod
metadata:
  name: test-2-1
spec:
  containers:
  - name: nginx
    image: nginx
    resources:
      requests:
        cpu: "500m"
        memory: "1Gi"
      limits:
        cpu: "500m"
        memory: "1Gi"

------------------------------------------------------------------------

kubectl apply -f test.yaml 

 

기존 노드 리소스 사용으로  신규 pod pending 상태로 진입 하면 cluster-autoscaler가 정상적으로 증설 하는지 확인

한쪽엔 kubectl -n kube-system logs -l app.kubernetes.io/name=aws-cluster-autoscaler -f  켜두면 편함