ericpuwang

K8s Iptables分析

Iptables

iptables 通常就是指linux上的防火墙,主要分为netfilter和iptables两个组件。netfilter为内核空间的组件,iptables为用户空间的组件,提供添加,删除查询防火墙规则的功能。

k8s的service通过iptables来做后端pod的转发和路由

环境准备

# 创建集群
kind create cluster --name iptables --image kindest/node:v1.23.5
# 安装traefik
# https://doc.traefik.io/traefik/getting-started/install-traefik/#use-the-helm-chart
helm repo add traefik https://helm.traefik.io/traefik
helm repo update
helm install traefik traefik/traefik
# 集群就绪后的iptables规则
root@iptables-control-plane:/# iptables-save
# Generated by iptables-save v1.8.4 on Wed Sep 28 06:24:53 2022
*mangle
:PREROUTING ACCEPT [1789547:356082101]
:INPUT ACCEPT [1789547:356082101]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [1784288:301003801]
:POSTROUTING ACCEPT [1784288:301003801]
:KUBE-KUBELET-CANARY - [0:0]
:KUBE-PROXY-CANARY - [0:0]
COMMIT
# Completed on Wed Sep 28 06:24:53 2022

# Generated by iptables-save v1.8.4 on Wed Sep 28 06:24:53 2022
*filter
:INPUT ACCEPT [1717:265397]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [1722:288167]
:KUBE-EXTERNAL-SERVICES - [0:0]
:KUBE-FIREWALL - [0:0]
:KUBE-FORWARD - [0:0]
:KUBE-KUBELET-CANARY - [0:0]
:KUBE-NODEPORTS - [0:0]
:KUBE-PROXY-CANARY - [0:0]
:KUBE-SERVICES - [0:0]
-A INPUT -m comment --comment "kubernetes health check service ports" -j KUBE-NODEPORTS
-A INPUT -m conntrack --ctstate NEW -m comment --comment "kubernetes externally-visible service portals" -j KUBE-EXTERNAL-SERVICES
-A INPUT -j KUBE-FIREWALL
-A FORWARD -m comment --comment "kubernetes forwarding rules" -j KUBE-FORWARD
-A FORWARD -m conntrack --ctstate NEW -m comment --comment "kubernetes service portals" -j KUBE-SERVICES
-A FORWARD -m conntrack --ctstate NEW -m comment --comment "kubernetes externally-visible service portals" -j KUBE-EXTERNAL-SERVICES
-A OUTPUT -m conntrack --ctstate NEW -m comment --comment "kubernetes service portals" -j KUBE-SERVICES
-A OUTPUT -j KUBE-FIREWALL
-A KUBE-FIREWALL -m comment --comment "kubernetes firewall for dropping marked packets" -m mark --mark 0x8000/0x8000 -j DROP
-A KUBE-FIREWALL ! -s 127.0.0.0/8 -d 127.0.0.0/8 -m comment --comment "block incoming localnet connections" -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP
-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
COMMIT
# Completed on Wed Sep 28 06:24:53 2022

# Generated by iptables-save v1.8.4 on Wed Sep 28 06:24:53 2022
*nat
:PREROUTING ACCEPT [1:60]
:INPUT ACCEPT [1:60]
:OUTPUT ACCEPT [26:1560]
:POSTROUTING ACCEPT [26:1560]
:DOCKER_OUTPUT - [0:0]
:DOCKER_POSTROUTING - [0:0]
:KIND-MASQ-AGENT - [0:0]
:KUBE-KUBELET-CANARY - [0:0]
:KUBE-MARK-DROP - [0:0]
:KUBE-MARK-MASQ - [0:0]
:KUBE-NODEPORTS - [0:0]
:KUBE-POSTROUTING - [0:0]
:KUBE-PROXY-CANARY - [0:0]
:KUBE-SEP-242VRRB6CJESMF7F - [0:0]
:KUBE-SEP-2KMBOYXVR33SDSC2 - [0:0]
:KUBE-SEP-6E7XQMQ4RAYOWTTM - [0:0]
:KUBE-SEP-OJCTP5LCEHQJ3D72 - [0:0]
:KUBE-SEP-PUHFDAMRBZWCPADU - [0:0]
:KUBE-SEP-SF3LG62VAE5ALYDV - [0:0]
:KUBE-SEP-WXWGHGKZOCNYRYI7 - [0:0]
:KUBE-SEP-ZP3FB6NMPNCO4VBJ - [0:0]
:KUBE-SEP-ZXMNUKOKXUTL2MK2 - [0:0]
:KUBE-SERVICES - [0:0]
:KUBE-SVC-3CRYVFGPLWYNKULQ - [0:0]
:KUBE-SVC-ERIFXISQEP7F7OF4 - [0:0]
:KUBE-SVC-JD5MR3NA4I4DYORP - [0:0]
:KUBE-SVC-NPX46M4PTMTKRN6Y - [0:0]
:KUBE-SVC-QWNV66JT3UNCF3AW - [0:0]
:KUBE-SVC-TCOU7JCQXEZGVUNU - [0:0]
-A PREROUTING -m comment --comment "kubernetes service portals" -j KUBE-SERVICES
-A PREROUTING -d 192.168.31.191/32 -j DOCKER_OUTPUT
-A OUTPUT -m comment --comment "kubernetes service portals" -j KUBE-SERVICES
-A OUTPUT -d 192.168.31.191/32 -j DOCKER_OUTPUT
-A POSTROUTING -m comment --comment "kubernetes postrouting rules" -j KUBE-POSTROUTING
-A POSTROUTING -d 192.168.31.191/32 -j DOCKER_POSTROUTING
-A POSTROUTING -m addrtype ! --dst-type LOCAL -m comment --comment "kind-masq-agent: ensure nat POSTROUTING directs all non-LOCAL destination traffic to our custom KIND-MASQ-AGENT chain" -j KIND-MASQ-AGENT
-A DOCKER_OUTPUT -d 192.168.31.191/32 -p tcp -m tcp --dport 53 -j DNAT --to-destination 127.0.0.11:42845
-A DOCKER_OUTPUT -d 192.168.31.191/32 -p udp -m udp --dport 53 -j DNAT --to-destination 127.0.0.11:50410
-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p tcp -m tcp --sport 42845 -j SNAT --to-source 192.168.31.191:53
-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p udp -m udp --sport 50410 -j SNAT --to-source 192.168.31.191:53
-A KIND-MASQ-AGENT -d 10.244.0.0/16 -m comment --comment "kind-masq-agent: local traffic is not subject to MASQUERADE" -j RETURN
-A KIND-MASQ-AGENT -m comment --comment "kind-masq-agent: outbound traffic is subject to MASQUERADE (must be last in chain)" -j MASQUERADE
-A KUBE-MARK-DROP -j MARK --set-xmark 0x8000/0x8000
-A KUBE-MARK-MASQ -j MARK --set-xmark 0x4000/0x4000
-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN
-A KUBE-POSTROUTING -j MARK --set-xmark 0x4000/0x0
-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --random-fully
-A KUBE-SEP-242VRRB6CJESMF7F -s 10.244.0.5/32 -m comment --comment "default/traefik:web" -j KUBE-MARK-MASQ
-A KUBE-SEP-242VRRB6CJESMF7F -p tcp -m comment --comment "default/traefik:web" -m tcp -j DNAT --to-destination 10.244.0.5:8000
-A KUBE-SEP-2KMBOYXVR33SDSC2 -s 10.244.0.5/32 -m comment --comment "default/traefik:websecure" -j KUBE-MARK-MASQ
-A KUBE-SEP-2KMBOYXVR33SDSC2 -p tcp -m comment --comment "default/traefik:websecure" -m tcp -j DNAT --to-destination 10.244.0.5:8443
-A KUBE-SEP-6E7XQMQ4RAYOWTTM -s 10.244.0.3/32 -m comment --comment "kube-system/kube-dns:dns" -j KUBE-MARK-MASQ
-A KUBE-SEP-6E7XQMQ4RAYOWTTM -p udp -m comment --comment "kube-system/kube-dns:dns" -m udp -j DNAT --to-destination 10.244.0.3:53
-A KUBE-SEP-OJCTP5LCEHQJ3D72 -s 172.18.0.5/32 -m comment --comment "default/kubernetes:https" -j KUBE-MARK-MASQ
-A KUBE-SEP-OJCTP5LCEHQJ3D72 -p tcp -m comment --comment "default/kubernetes:https" -m tcp -j DNAT --to-destination 172.18.0.5:6443
-A KUBE-SEP-PUHFDAMRBZWCPADU -s 10.244.0.4/32 -m comment --comment "kube-system/kube-dns:metrics" -j KUBE-MARK-MASQ
-A KUBE-SEP-PUHFDAMRBZWCPADU -p tcp -m comment --comment "kube-system/kube-dns:metrics" -m tcp -j DNAT --to-destination 10.244.0.4:9153
-A KUBE-SEP-SF3LG62VAE5ALYDV -s 10.244.0.4/32 -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-MARK-MASQ
-A KUBE-SEP-SF3LG62VAE5ALYDV -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp" -m tcp -j DNAT --to-destination 10.244.0.4:53
-A KUBE-SEP-WXWGHGKZOCNYRYI7 -s 10.244.0.4/32 -m comment --comment "kube-system/kube-dns:dns" -j KUBE-MARK-MASQ
-A KUBE-SEP-WXWGHGKZOCNYRYI7 -p udp -m comment --comment "kube-system/kube-dns:dns" -m udp -j DNAT --to-destination 10.244.0.4:53
-A KUBE-SEP-ZP3FB6NMPNCO4VBJ -s 10.244.0.3/32 -m comment --comment "kube-system/kube-dns:metrics" -j KUBE-MARK-MASQ
-A KUBE-SEP-ZP3FB6NMPNCO4VBJ -p tcp -m comment --comment "kube-system/kube-dns:metrics" -m tcp -j DNAT --to-destination 10.244.0.3:9153
-A KUBE-SEP-ZXMNUKOKXUTL2MK2 -s 10.244.0.3/32 -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-MARK-MASQ
-A KUBE-SEP-ZXMNUKOKXUTL2MK2 -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp" -m tcp -j DNAT --to-destination 10.244.0.3:53
-A KUBE-SERVICES -d 10.96.237.231/32 -p tcp -m comment --comment "default/traefik:web cluster IP" -m tcp --dport 80 -j KUBE-SVC-3CRYVFGPLWYNKULQ
-A KUBE-SERVICES -d 10.96.237.231/32 -p tcp -m comment --comment "default/traefik:websecure cluster IP" -m tcp --dport 443 -j KUBE-SVC-QWNV66JT3UNCF3AW
-A KUBE-SERVICES -d 10.96.0.1/32 -p tcp -m comment --comment "default/kubernetes:https cluster IP" -m tcp --dport 443 -j KUBE-SVC-NPX46M4PTMTKRN6Y
-A KUBE-SERVICES -d 10.96.0.10/32 -p udp -m comment --comment "kube-system/kube-dns:dns cluster IP" -m udp --dport 53 -j KUBE-SVC-TCOU7JCQXEZGVUNU
-A KUBE-SERVICES -d 10.96.0.10/32 -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp cluster IP" -m tcp --dport 53 -j KUBE-SVC-ERIFXISQEP7F7OF4
-A KUBE-SERVICES -d 10.96.0.10/32 -p tcp -m comment --comment "kube-system/kube-dns:metrics cluster IP" -m tcp --dport 9153 -j KUBE-SVC-JD5MR3NA4I4DYORP
-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS
-A KUBE-SVC-3CRYVFGPLWYNKULQ ! -s 10.244.0.0/16 -d 10.96.237.231/32 -p tcp -m comment --comment "default/traefik:web cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ
-A KUBE-SVC-3CRYVFGPLWYNKULQ -m comment --comment "default/traefik:web" -j KUBE-SEP-242VRRB6CJESMF7F
-A KUBE-SVC-ERIFXISQEP7F7OF4 ! -s 10.244.0.0/16 -d 10.96.0.10/32 -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp cluster IP" -m tcp --dport 53 -j KUBE-MARK-MASQ
-A KUBE-SVC-ERIFXISQEP7F7OF4 -m comment --comment "kube-system/kube-dns:dns-tcp" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-ZXMNUKOKXUTL2MK2
-A KUBE-SVC-ERIFXISQEP7F7OF4 -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-SEP-SF3LG62VAE5ALYDV
-A KUBE-SVC-JD5MR3NA4I4DYORP ! -s 10.244.0.0/16 -d 10.96.0.10/32 -p tcp -m comment --comment "kube-system/kube-dns:metrics cluster IP" -m tcp --dport 9153 -j KUBE-MARK-MASQ
-A KUBE-SVC-JD5MR3NA4I4DYORP -m comment --comment "kube-system/kube-dns:metrics" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-ZP3FB6NMPNCO4VBJ
-A KUBE-SVC-JD5MR3NA4I4DYORP -m comment --comment "kube-system/kube-dns:metrics" -j KUBE-SEP-PUHFDAMRBZWCPADU
-A KUBE-SVC-NPX46M4PTMTKRN6Y ! -s 10.244.0.0/16 -d 10.96.0.1/32 -p tcp -m comment --comment "default/kubernetes:https cluster IP" -m tcp --dport 443 -j KUBE-MARK-MASQ
-A KUBE-SVC-NPX46M4PTMTKRN6Y -m comment --comment "default/kubernetes:https" -j KUBE-SEP-OJCTP5LCEHQJ3D72
-A KUBE-SVC-QWNV66JT3UNCF3AW ! -s 10.244.0.0/16 -d 10.96.237.231/32 -p tcp -m comment --comment "default/traefik:websecure cluster IP" -m tcp --dport 443 -j KUBE-MARK-MASQ
-A KUBE-SVC-QWNV66JT3UNCF3AW -m comment --comment "default/traefik:websecure" -j KUBE-SEP-2KMBOYXVR33SDSC2
-A KUBE-SVC-TCOU7JCQXEZGVUNU ! -s 10.244.0.0/16 -d 10.96.0.10/32 -p udp -m comment --comment "kube-system/kube-dns:dns cluster IP" -m udp --dport 53 -j KUBE-MARK-MASQ
-A KUBE-SVC-TCOU7JCQXEZGVUNU -m comment --comment "kube-system/kube-dns:dns" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-6E7XQMQ4RAYOWTTM
-A KUBE-SVC-TCOU7JCQXEZGVUNU -m comment --comment "kube-system/kube-dns:dns" -j KUBE-SEP-WXWGHGKZOCNYRYI7
COMMIT
# Completed on Wed Sep 28 06:24:53 2022

service

有如下的映射关系

clusterIp:port podIp:port
10.96.237.231:80 10.244.0.5:8000
# kubectl describe svc traefik
Name:              traefik
Namespace:         default
Labels:            app.kubernetes.io/instance=traefik
                   app.kubernetes.io/managed-by=Helm
                   app.kubernetes.io/name=traefik
                   helm.sh/chart=traefik-10.25.0
Annotations:       meta.helm.sh/release-name: traefik
                   meta.helm.sh/release-namespace: default
Selector:          app.kubernetes.io/instance=traefik,app.kubernetes.io/name=traefik
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                10.96.237.231
IPs:               10.96.237.231
Port:              web  80/TCP
TargetPort:        web/TCP
Endpoints:         10.244.0.5:8000
Port:              websecure  443/TCP
TargetPort:        websecure/TCP
Endpoints:         10.244.0.5:8443
# kubectl get svc traefik -owide
NAME      TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE     SELECTOR
traefik   ClusterIP   10.96.237.231   <none>        80/TCP,443/TCP   3m25s   app.kubernetes.io/instance=traefik,app.kubernetes.io/name=traefik

iptables

入网流量

1. nat表

PREROUTING -> KUBE-SERVICES -> KUBE-SVC-3CRYVFGPLWYNKULQ -> KUBE-SEP-242VRRB6CJESMF7F -> DNAT

# PREROUTING
Chain PREROUTING (policy ACCEPT 13 packets, 780 bytes)
 pkts bytes target     prot opt in     out     source               destination
   98  6678 KUBE-SERVICES  all  --  *      *       0.0.0.0/0            0.0.0.0/0            /* kubernetes service portals */
    6   490 DOCKER_OUTPUT  all  --  *      *       0.0.0.0/0            192.168.31.191
# KUBE-SERVICES
Chain KUBE-SERVICES (2 references)
 pkts bytes target     prot opt in     out     source               destination
    0     0 KUBE-SVC-3CRYVFGPLWYNKULQ  tcp  --  *      *       0.0.0.0/0            10.96.237.231        /* default/traefik:web cluster IP */ tcp dpt:80
    0     0 KUBE-SVC-QWNV66JT3UNCF3AW  tcp  --  *      *       0.0.0.0/0            10.96.237.231        /* default/traefik:websecure cluster IP */ tcp dpt:443
    0     0 KUBE-SVC-NPX46M4PTMTKRN6Y  tcp  --  *      *       0.0.0.0/0            10.96.0.1            /* default/kubernetes:https cluster IP */ tcp dpt:443
   24  2108 KUBE-SVC-TCOU7JCQXEZGVUNU  udp  --  *      *       0.0.0.0/0            10.96.0.10           /* kube-system/kube-dns:dns cluster IP */ udp dpt:53
    0     0 KUBE-SVC-ERIFXISQEP7F7OF4  tcp  --  *      *       0.0.0.0/0            10.96.0.10           /* kube-system/kube-dns:dns-tcp cluster IP */ tcp dpt:53
    0     0 KUBE-SVC-JD5MR3NA4I4DYORP  tcp  --  *      *       0.0.0.0/0            10.96.0.10           /* kube-system/kube-dns:metrics cluster IP */ tcp dpt:9153
 1233 73980 KUBE-NODEPORTS  all  --  *      *       0.0.0.0/0            0.0.0.0/0            /* kubernetes service nodeports; NOTE: this must be the last rule in this chain */ ADDRTYPE match dst-type LOCAL
# KUBE-SVC-3CRYVFGPLWYNKULQ
Chain KUBE-SVC-3CRYVFGPLWYNKULQ (1 references)
 pkts bytes target     prot opt in     out     source               destination
    0     0 KUBE-MARK-MASQ  tcp  --  *      *      !10.244.0.0/16        10.96.237.231        /* default/traefik:web cluster IP */ tcp dpt:80
    0     0 KUBE-SEP-242VRRB6CJESMF7F  all  --  *      *       0.0.0.0/0            0.0.0.0/0            /* default/traefik:web */
# KUBE-SEP-242VRRB6CJESMF7F
Chain KUBE-SEP-242VRRB6CJESMF7F (1 references)
 pkts bytes target     prot opt in     out     source               destination
    0     0 KUBE-MARK-MASQ  all  --  *      *       10.244.0.5           0.0.0.0/0            /* default/traefik:web */
    0     0 DNAT       tcp  --  *      *       0.0.0.0/0            0.0.0.0/0            /* default/traefik:web */ tcp to:10.244.0.5:8000

2. filter表

INPUT -> KUBE-NODEPORTS -> KUBE-EXTERNAL-SERVICES -> KUBE-FIREWALL

# INPUT
Chain INPUT (policy ACCEPT 166K packets, 27M bytes)
 pkts bytes target     prot opt in     out     source               destination
1954K  383M KUBE-NODEPORTS  all  --  *      *       0.0.0.0/0            0.0.0.0/0            /* kubernetes health check service ports */
17707 1063K KUBE-EXTERNAL-SERVICES  all  --  *      *       0.0.0.0/0            0.0.0.0/0            ctstate NEW /* kubernetes externally-visible service portals */
1975K  389M KUBE-FIREWALL  all  --  *      *       0.0.0.0/0            0.0.0.0/0
# KUBE-NODEPORTS
Chain KUBE-NODEPORTS (1 references)
 pkts bytes target     prot opt in     out     source               destination
# KUBE-EXTERNAL-SERVICES
Chain KUBE-EXTERNAL-SERVICES (2 references)
 pkts bytes target     prot opt in     out     source               destination
# KUBE-FIREWALL
Chain KUBE-FIREWALL (2 references)
 pkts bytes target     prot opt in     out     source               destination
    0     0 DROP       all  --  *      *       0.0.0.0/0            0.0.0.0/0            /* kubernetes firewall for dropping marked packets */ mark match 0x8000/0x8000
    0     0 DROP       all  --  *      *      !127.0.0.0/8          127.0.0.0/8          /* block incoming localnet connections */ ! ctstate RELATED,ESTABLISHED,DNAT

出网流量

1. nat表

OUTPUT -> KUBE-SERVICES

ps: 出网流量不会命中KUBE-SERVICES链中的规则

# OUTPUT
Chain OUTPUT (policy ACCEPT 3020 packets, 181K bytes)
 pkts bytes target     prot opt in     out     source               destination
23756 1426K KUBE-SERVICES  all  --  *      *       0.0.0.0/0            0.0.0.0/0            /* kubernetes service portals */
   48  3294 DOCKER_OUTPUT  all  --  *      *       0.0.0.0/0            192.168.31.191

2. filter表

OUTPUT -> KUBE-SERVICES -> KUBE-FIREWALL

# OUTPUT
Chain OUTPUT (policy ACCEPT 166K packets, 28M bytes)
 pkts bytes target     prot opt in     out     source               destination
22875 1373K KUBE-SERVICES  all  --  *      *       0.0.0.0/0            0.0.0.0/0            ctstate NEW /* kubernetes service portals */
1970K  335M KUBE-FIREWALL  all  --  *      *       0.0.0.0/0            0.0.0.0/0
# KUBE-SERVICES
Chain KUBE-SERVICES (2 references)
 pkts bytes target     prot opt in     out     source               destination
# KUBE-FIREWALL
Chain KUBE-FIREWALL (2 references)
 pkts bytes target     prot opt in     out     source               destination
    0     0 DROP       all  --  *      *       0.0.0.0/0            0.0.0.0/0            /* kubernetes firewall for dropping marked packets */ mark match 0x8000/0x8000
    0     0 DROP       all  --  *      *      !127.0.0.0/8          127.0.0.0/8          /* block incoming localnet connections */ ! ctstate RELATED,ESTABLISHED,DNAT

3. nat表

POSTROUTING -> KUBE-POSTROUTING -> MASQUERADE

# POSTROUTING
Chain POSTROUTING (policy ACCEPT 3038 packets, 183K bytes)
 pkts bytes target     prot opt in     out     source               destination
23934 1438K KUBE-POSTROUTING  all  --  *      *       0.0.0.0/0            0.0.0.0/0            /* kubernetes postrouting rules */
    0     0 DOCKER_POSTROUTING  all  --  *      *       0.0.0.0/0            192.168.31.191
 5518  332K KIND-MASQ-AGENT  all  --  *      *       0.0.0.0/0            0.0.0.0/0            ADDRTYPE match dst-type !LOCAL /* kind-masq-agent: ensure nat POSTROUTING directs all non-LOCAL destination traffic to our custom KIND-MASQ-AGENT chain */
# KUBE-POSTROUTING
Chain KUBE-POSTROUTING (1 references)
 pkts bytes target     prot opt in     out     source               destination
 3050  184K RETURN     all  --  *      *       0.0.0.0/0            0.0.0.0/0            mark match ! 0x4000/0x4000
    0     0 MARK       all  --  *      *       0.0.0.0/0            0.0.0.0/0            MARK xor 0x4000
    0     0 MASQUERADE  all  --  *      *       0.0.0.0/0            0.0.0.0/0            /* kubernetes service traffic requiring SNAT */ random-fully
# DOCKER_POSTROUTING
Chain DOCKER_POSTROUTING (1 references)
 pkts bytes target     prot opt in     out     source               destination
    0     0 SNAT       tcp  --  *      *       127.0.0.11           0.0.0.0/0            tcp spt:42845 to:192.168.31.191:53
    0     0 SNAT       udp  --  *      *       127.0.0.11           0.0.0.0/0            udp spt:50410 to:192.168.31.191:53
# KIND-MASQ-AGENT
Chain KIND-MASQ-AGENT (1 references)
 pkts bytes target     prot opt in     out     source               destination
 5422  326K RETURN     all  --  *      *       0.0.0.0/0            10.244.0.0/16        /* kind-masq-agent: local traffic is not subject to MASQUERADE */
   96  6326 MASQUERADE  all  --  *      *       0.0.0.0/0            0.0.0.0/0            /* kind-masq-agent: outbound traffic is subject to MASQUERADE (must be last in chain) */