Kubernetes Networking & AWS EKS Networking: The Enterprise-Grade Deep Dive

helm repo add cilium https://helm.cilium.io/
helm repo update

helm install cilium cilium/cilium \
  --namespace kube-system \
  --set kubeProxyReplacement=strict \
  --set k8sServiceHost=<API_SERVER_ENDPOINT> \
  --set k8sServicePort=443 \
  --set ipam.mode=kubernetes \
  --set nodeinit.enabled=true

Pod eth0  ←→  veth pair  ←→  node bridge  ←→  routing table  ←→  other pod

# Get the pod's node and container ID
kubectl get pod nginx-pod -o wide
kubectl get pod nginx-pod \
  -o jsonpath='{.status.containerStatuses[0].containerID}'

# On the node, inspect the network namespace
PID=$(crictl inspect <CONTAINER_ID> | jq '.info.pid')
nsenter -t $PID -n ip addr
nsenter -t $PID -n ip route

# Verify connectivity between pods
kubectl exec -it nginx-pod -- curl http://<other-pod-ip>:8080

# ClusterIP – internal only
apiVersion: v1
kind: Service
metadata:
  name: backend-svc
  namespace: production
spec:
  type: ClusterIP
  selector:
    app: backend
  ports:
    - protocol: TCP
      port: 80
      targetPort: 8080

---
# NodePort – opens a port on every node
apiVersion: v1
kind: Service
metadata:
  name: backend-nodeport
spec:
  type: NodePort
  selector:
    app: backend
  ports:
    - port: 80
      targetPort: 8080
      nodePort: 30080   # Range: 30000-32767

---
# LoadBalancer – provisions an external load balancer (NLB/ALB on EKS)
apiVersion: v1
kind: Service
metadata:
  name: backend-lb
  annotations:
    service.beta.kubernetes.io/aws-load-balancer-type: "external"
    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"
spec:
  type: LoadBalancer
  selector:
    app: backend
  ports:
    - port: 443
      targetPort: 8443

apiVersion: v1
kind: Service
metadata:
  name: cassandra
spec:
  clusterIP: None      # Headless: no VIP, DNS returns pod IPs directly
  selector:
    app: cassandra
  ports:
    - port: 9042

# Check current kube-proxy mode on EKS
kubectl get configmap kube-proxy-config -n kube-system -o yaml | grep mode

# Switch to IPVS mode (more scalable, O(1) service lookup)
kubectl edit configmap kube-proxy-config -n kube-system
# Change: mode: "iptables" → mode: "ipvs"

<service-name>.<namespace>.svc.cluster.local

# Test DNS resolution from a debug pod
kubectl run debug --image=busybox --rm -it -- nslookup backend-svc.production.svc.cluster.local

# Inspect CoreDNS configuration
kubectl get configmap coredns -n kube-system -o yaml

# Scale CoreDNS for high-traffic clusters
kubectl scale deployment coredns -n kube-system --replicas=4

apiVersion: v1
kind: ConfigMap
metadata:
  name: coredns
  namespace: kube-system
data:
  Corefile: |
    .:53 {
        errors
        health { lameduck 5s }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
            pods insecure
            fallthrough in-addr.arpa ip6.arpa
        }
        prometheus :9153
        forward . /etc/resolv.conf {
            max_concurrent 1000
        }
        cache 30
        loop
        reload
        loadbalance
    }
    internal.corp:53 {
        forward . 10.0.0.2
    }
}

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-all
  namespace: production
spec:
  podSelector: {}      # Selects ALL pods in namespace
  policyTypes:
    - Ingress
    - Egress

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-frontend-to-backend
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: backend
  policyTypes:
    - Ingress
  ingress:
    - from:
        - podSelector:
            matchLabels:
              app: frontend
      ports:
        - protocol: TCP
          port: 8080

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-db-egress
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: backend
  policyTypes:
    - Egress
  egress:
    - to:
        - ipBlock:
            cidr: 10.20.0.0/24   # RDS subnet CIDR
      ports:
        - protocol: TCP
          port: 3306
    - to: []                       # Allow DNS
      ports:
        - protocol: UDP
          port: 53

helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
helm install ingress-nginx ingress-nginx/ingress-nginx \
  --namespace ingress-nginx --create-namespace \
  --set controller.service.type=LoadBalancer \
  --set controller.replicaCount=3

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: app-ingress
  namespace: production
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    nginx.ingress.kubernetes.io/proxy-body-size: "50m"
spec:
  ingressClassName: nginx
  tls:
    - hosts:
        - api.example.com
      secretName: api-tls-secret
  rules:
    - host: api.example.com
      http:
        paths:
          - path: /v1
            pathType: Prefix
            backend:
              service:
                name: api-svc
                port:
                  number: 80
          - path: /static
            pathType: Prefix
            backend:
              service:
                name: static-svc
                port:
                  number: 80

# GatewayClass – provisioned by cluster admin
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
  name: aws-gateway
spec:
  controllerName: eks.amazonaws.com/gateway-controller

---
# Gateway – owned by network team
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: prod-gateway
  namespace: production
spec:
  gatewayClassName: aws-gateway
  listeners:
    - name: https
      protocol: HTTPS
      port: 443
      tls:
        mode: Terminate
        certificateRefs:
          - name: prod-tls-cert

---
# HTTPRoute – owned by app team
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: api-route
  namespace: production
spec:
  parentRefs:
    - name: prod-gateway
  hostnames:
    - "api.example.com"
  rules:
    - matches:
        - path:
            type: PathPrefix
            value: /v1
      backendRefs:
        - name: api-svc
          port: 80
          weight: 100

# Install Istio on EKS
curl -L https://istio.io/downloadIstio | sh -
istioctl install --set profile=production -y

# Enable automatic sidecar injection for a namespace
kubectl label namespace production istio-injection=enabled

# Verify mesh
kubectl get pods -n production   # Each pod should show 2/2 READY (app + envoy)
istioctl analyze -n production

apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: backend-vs
  namespace: production
spec:
  hosts:
    - backend-svc
  http:
    - route:
        - destination:
            host: backend-svc
            subset: v1
          weight: 90
        - destination:
            host: backend-svc
            subset: v2
          weight: 10

---
apiVersion: networking.istio.io/v1alpha3
kind: DestinationRule
metadata:
  name: backend-dr
  namespace: production
spec:
  host: backend-svc
  trafficPolicy:
    tls:
      mode: ISTIO_MUTUAL    # Enforce mTLS
  subsets:
    - name: v1
      labels:
        version: v1
    - name: v2
      labels:
        version: v2

VPC: 10.0.0.0/16
├── Public Subnets (one per AZ) – NAT Gateways, Load Balancers
│   ├── 10.0.1.0/24  (us-east-1a)
│   ├── 10.0.2.0/24  (us-east-1b)
│   └── 10.0.3.0/24  (us-east-1c)
├── Private Subnets (Worker Nodes)
│   ├── 10.0.11.0/24 (us-east-1a)
│   ├── 10.0.12.0/24 (us-east-1b)
│   └── 10.0.13.0/24 (us-east-1c)
└── Pod Subnets (large /19 blocks for IP density)
    ├── 10.0.128.0/19 (us-east-1a)
    ├── 10.0.160.0/19 (us-east-1b)
    └── 10.0.192.0/19 (us-east-1c)

# Tag public subnets for external ALB/NLB
aws ec2 create-tags --resources subnet-xxxxxx \
  --tags Key=kubernetes.io/role/elb,Value=1 \
         Key=kubernetes.io/cluster/<cluster-name>,Value=shared

# Tag private subnets for internal LBs
aws ec2 create-tags --resources subnet-yyyyyy \
  --tags Key=kubernetes.io/role/internal-elb,Value=1 \
         Key=kubernetes.io/cluster/<cluster-name>,Value=shared

Max Pods = (Max ENIs × (IPs per ENI - 1)) + 2

# Check current VPC CNI version
kubectl describe daemonset aws-node -n kube-system | grep Image

# Check IP allocation on a node
kubectl get node ip-10-0-11-50.ec2.internal \
  -o jsonpath='{.metadata.annotations.vpc\.amazonaws\.com/node-capacity}'

# Enable prefix delegation (multiplies IP capacity by 16)
kubectl set env daemonset aws-node -n kube-system \
  ENABLE_PREFIX_DELEGATION=true \
  WARM_PREFIX_TARGET=1

# Step 1: Enable custom networking
kubectl set env daemonset aws-node -n kube-system \
  AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG=true \
  ENI_CONFIG_LABEL_DEF=topology.kubernetes.io/zone

# Step 2: Create ENIConfig per AZ
cat <<EOF | kubectl apply -f -
apiVersion: crd.k8s.amazonaws.com/v1alpha1
kind: ENIConfig
metadata:
  name: us-east-1a
spec:
  subnet: subnet-pod-subnet-1a-id      # Pod-dedicated subnet
  securityGroups:
    - sg-xxxxxxxxxxxxxxxxx
EOF

# terraform/main.tf

terraform {
  required_providers {
    aws        = { source = "hashicorp/aws", version = "~> 5.0" }
    kubernetes = { source = "hashicorp/kubernetes", version = "~> 2.0" }
    helm       = { source = "hashicorp/helm", version = "~> 2.0" }
  }
}

provider "aws" {
  region = var.region
}

# ── VPC ────────────────────────────────────────────────────────────────────
module "vpc" {
  source  = "terraform-aws-modules/vpc/aws"
  version = "~> 5.0"

  name = "${var.cluster_name}-vpc"
  cidr = "10.0.0.0/16"

  azs             = ["us-east-1a", "us-east-1b", "us-east-1c"]
  private_subnets = ["10.0.11.0/24", "10.0.12.0/24", "10.0.13.0/24"]
  public_subnets  = ["10.0.1.0/24",  "10.0.2.0/24",  "10.0.3.0/24"]
  intra_subnets   = ["10.0.128.0/19","10.0.160.0/19","10.0.192.0/19"] # Pod subnets

  enable_nat_gateway     = true
  single_nat_gateway     = false   # One per AZ for HA
  enable_vpn_gateway     = false
  enable_dns_hostnames   = true
  enable_dns_support     = true

  public_subnet_tags = {
    "kubernetes.io/role/elb"                        = "1"
    "kubernetes.io/cluster/${var.cluster_name}"     = "shared"
  }

  private_subnet_tags = {
    "kubernetes.io/role/internal-elb"               = "1"
    "kubernetes.io/cluster/${var.cluster_name}"     = "shared"
  }
}

# ── EKS Cluster ───────────────────────────────────────────────────────────
module "eks" {
  source  = "terraform-aws-modules/eks/aws"
  version = "~> 20.0"

  cluster_name    = var.cluster_name
  cluster_version = "1.30"

  cluster_endpoint_public_access       = true
  cluster_endpoint_private_access      = true
  cluster_endpoint_public_access_cidrs = ["YOUR_BASTION_IP/32"]   # Restrict API access

  vpc_id                   = module.vpc.vpc_id
  subnet_ids               = module.vpc.private_subnets
  control_plane_subnet_ids = module.vpc.intra_subnets

  # EKS Addons
  cluster_addons = {
    coredns = {
      most_recent = true
      configuration_values = jsonencode({
        replicaCount = 3
        resources = {
          limits   = { cpu = "200m", memory = "256Mi" }
          requests = { cpu = "100m", memory = "128Mi" }
        }
      })
    }
    kube-proxy = { most_recent = true }
    vpc-cni = {
      most_recent = true
      configuration_values = jsonencode({
        env = {
          ENABLE_PREFIX_DELEGATION = "true"
          WARM_PREFIX_TARGET       = "1"
        }
      })
    }
    aws-ebs-csi-driver = { most_recent = true }
  }

  # Managed Node Groups
  eks_managed_node_groups = {
    system = {
      name           = "system-ng"
      instance_types = ["m5.xlarge"]
      min_size       = 2
      max_size       = 4
      desired_size   = 3

      subnet_ids = module.vpc.private_subnets

      labels = { role = "system" }
      taints = [{
        key    = "CriticalAddonsOnly"
        value  = "true"
        effect = "NO_SCHEDULE"
      }]
    }

    application = {
      name           = "app-ng"
      instance_types = ["m5.2xlarge", "m5.4xlarge"]
      capacity_type  = "SPOT"
      min_size       = 3
      max_size       = 30
      desired_size   = 6

      subnet_ids = module.vpc.private_subnets

      labels = { role = "application" }
    }
  }

  # Security Groups for Pods (Security Groups for Pods feature)
  node_security_group_additional_rules = {
    ingress_self_all = {
      description = "Node to node all ports/protocols"
      protocol    = "-1"
      from_port   = 0
      to_port     = 0
      type        = "ingress"
      self        = true
    }
    egress_all = {
      description      = "Node all egress"
      protocol         = "-1"
      from_port        = 0
      to_port          = 0
      type             = "egress"
      cidr_blocks      = ["0.0.0.0/0"]
    }
  }
}

# ── AWS Load Balancer Controller ──────────────────────────────────────────
module "aws_lbc_irsa" {
  source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
  version = "~> 5.0"

  role_name                              = "${var.cluster_name}-aws-lbc"
  attach_load_balancer_controller_policy = true

  oidc_providers = {
    ex = {
      provider_arn               = module.eks.oidc_provider_arn
      namespace_service_accounts = ["kube-system:aws-load-balancer-controller"]
    }
  }
}

resource "helm_release" "aws_lbc" {
  name       = "aws-load-balancer-controller"
  repository = "https://aws.github.io/eks-charts"
  chart      = "aws-load-balancer-controller"
  namespace  = "kube-system"
  version    = "1.8.0"

  set { name = "clusterName";                     value = var.cluster_name }
  set { name = "serviceAccount.create";           value = "true" }
  set { name = "serviceAccount.name";             value = "aws-load-balancer-controller" }
  set { name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
        value = module.aws_lbc_irsa.iam_role_arn }
  set { name = "replicaCount";                    value = "2" }
  set { name = "vpcId";                           value = module.vpc.vpc_id }
  set { name = "region";                          value = var.region }
}

# ALB Ingress with HTTPS, WAF, and target type IP
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: production-ingress
  namespace: production
  annotations:
    alb.ingress.kubernetes.io/scheme: internet-facing
    alb.ingress.kubernetes.io/target-type: ip          # Route to Pod IPs directly
    alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS": 443}]'
    alb.ingress.kubernetes.io/ssl-redirect: "443"
    alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:ACCOUNT:certificate/CERT_ID
    alb.ingress.kubernetes.io/wafv2-acl-arn: arn:aws:wafv2:us-east-1:ACCOUNT:regional/webacl/NAME/ID
    alb.ingress.kubernetes.io/healthcheck-path: /health
    alb.ingress.kubernetes.io/healthcheck-interval-seconds: "15"
    alb.ingress.kubernetes.io/success-codes: "200,201"
    alb.ingress.kubernetes.io/load-balancer-attributes: >-
      idle_timeout.timeout_seconds=120,
      routing.http2.enabled=true,
      access_logs.s3.enabled=true,
      access_logs.s3.bucket=my-alb-logs,
      access_logs.s3.prefix=eks-prod
spec:
  ingressClassName: alb
  rules:
    - host: api.example.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: api-svc
                port:
                  number: 80

# Enable Security Groups for Pods on VPC CNI
kubectl set env daemonset aws-node -n kube-system \
  ENABLE_POD_ENI=true

# Verify trunk ENI support on nodes
kubectl describe node <node-name> | grep vpc.amazonaws.com/has-trunk-attached

# SecurityGroupPolicy CRD – assign SG to specific pods
apiVersion: vpcresources.k8s.aws/v1beta1
kind: SecurityGroupPolicy
metadata:
  name: backend-sgp
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: backend
  securityGroups:
    groupIds:
      - sg-backend-to-rds-id       # SG that allows 3306 to RDS
      - sg-backend-to-redis-id     # SG that allows 6379 to ElastiCache

# ── Cluster Creation with eksctl ──────────────────────────────────────────
cat <<EOF > cluster.yaml
apiVersion: eksctl.io/v1alpha5
kind: ClusterConfig
metadata:
  name: prod-cluster
  region: us-east-1
  version: "1.30"

vpc:
  cidr: 10.0.0.0/16
  clusterEndpoints:
    publicAccess:  true
    privateAccess: true
  publicAccessCIDRs: ["YOUR_IP/32"]

addons:
  - name: vpc-cni
    version: latest
    configurationValues: |
      {"env":{"ENABLE_PREFIX_DELEGATION":"true","WARM_PREFIX_TARGET":"1"}}
  - name: coredns
    version: latest
  - name: kube-proxy
    version: latest
  - name: aws-ebs-csi-driver
    version: latest

managedNodeGroups:
  - name: system
    instanceType: m5.xlarge
    desiredCapacity: 3
    minSize: 2
    maxSize: 6
    privateNetworking: true
    subnets:
      - private-us-east-1a
      - private-us-east-1b
      - private-us-east-1c
    taints:
      - key: CriticalAddonsOnly
        value: "true"
        effect: NoSchedule

  - name: application
    instanceTypes: ["m5.2xlarge", "m5.4xlarge"]
    desiredCapacity: 6
    minSize: 3
    maxSize: 30
    privateNetworking: true
    spot: true
EOF

eksctl create cluster -f cluster.yaml

# ── Network Diagnostics ───────────────────────────────────────────────────
# Check all node IPs and ENI capacity
kubectl get nodes -o custom-columns=\
'NAME:.metadata.name,IP:.status.addresses.address,\
MAX-PODS:.status.allocatable.pods'

# Describe VPC CNI daemonset
kubectl describe daemonset aws-node -n kube-system

# Check CNI logs for IP allocation issues
kubectl logs -n kube-system -l k8s-app=aws-node --tail=100

# Verify pod networking end-to-end
kubectl run nettest --image=nicolaka/netshoot --rm -it -- \
  traceroute 10.0.11.100     # Trace route to another pod IP

# Check network policies in effect
kubectl get networkpolicies -A
kubectl describe networkpolicy default-deny-all -n production

# Install CloudWatch agent via Helm
helm repo add aws-observability https://aws-observability.github.io/helm-charts
helm install aws-cloudwatch-metrics aws-observability/aws-cloudwatch-metrics \
  --namespace amazon-cloudwatch --create-namespace \
  --set clusterName=prod-cluster

# In Terraform – enable VPC flow logs to S3
resource "aws_flow_log" "eks_vpc" {
  vpc_id          = module.vpc.vpc_id
  traffic_type    = "ALL"
  iam_role_arn    = aws_iam_role.flow_logs.arn
  log_destination = aws_s3_bucket.flow_logs.arn
  log_destination_type = "s3"

  log_format = "$${version} $${account-id} $${vpc-id} $${subnet-id} $${srcaddr} $${dstaddr} $${srcport} $${dstport} $${protocol} $${packets} $${bytes} $${action} $${log-status}"
}

# Enable Hubble UI
helm upgrade cilium cilium/cilium -n kube-system \
  --reuse-values \
  --set hubble.relay.enabled=true \
  --set hubble.ui.enabled=true

# Port-forward Hubble UI
kubectl port-forward -n kube-system svc/hubble-ui 12000:80 &
# Open http://localhost:12000 for visual flow maps

# CLI: watch live network flows
hubble observe --namespace production --follow

# Create IPv6 EKS cluster with eksctl
cat <<EOF | eksctl create cluster -f -
apiVersion: eksctl.io/v1alpha5
kind: ClusterConfig
metadata:
  name: ipv6-cluster
  region: us-east-1
  version: "1.30"
kubernetesNetworkConfig:
  ipFamily: IPv6
managedNodeGroups:
  - name: ng-ipv6
    instanceType: m5.xlarge
    desiredCapacity: 3
EOF

# Pod can't reach external internet
kubectl exec -it <pod> -- curl -v https://example.com
# Check: NAT Gateway route in private subnet route table

# DNS resolution failing
kubectl exec -it <pod> -- nslookup kubernetes.default
kubectl logs -n kube-system -l k8s-app=kube-dns --tail=50
# Check: CoreDNS pod health, security group allows UDP/TCP 53 from nodes

# Service not reachable
kubectl get endpoints <service-name> -n <namespace>
# If endpoints list is empty: label selector on Service doesn't match pod labels

# ALB not provisioned
kubectl describe ingress <ingress-name> -n <namespace>
kubectl logs -n kube-system -l app.kubernetes.io/name=aws-load-balancer-controller
# Check: subnet tags, IRSA role permissions, security group for ALB

# IP exhaustion on nodes
kubectl describe node <node> | grep -A5 "Allocated resources"
# Fix: enable prefix delegation or add nodes with higher ENI capacity

# NetworkPolicy blocking traffic unexpectedly
kubectl get networkpolicies -n <namespace>
# Use: kubectl exec debug pod and curl; use Cilium Hubble or "kubectl-netpol" for visualization

# Pods stuck in ContainerCreating (CNI error)
kubectl describe pod <pod-name>
kubectl logs -n kube-system aws-node-<node-id>
# Common: WARM_IP_TARGET exhausted, ENI attach limit reached