【发布时间】：2018-09-03 21:57:52
【问题描述】：
我正在运行一个 4 节点集群（在数据中心虚拟机上）：通过 2 个服务公开的 2 个 pod * 通过端口 5432 上的服务 postgresql-k8s-service 公开的第一个服务 postgresql rinning Fine。 * 第二个服务 Artifcatory 基本上是一个 tomcat 容器，它试图连接到这个 postgresql，不幸的是，pod 无法连接到服务，不确定发生了什么。
错误：在 30 秒内等待数据库 postgresql 在 postgresql-k8s-service/5432 上准备就绪
Login to Artifactory pod and run " ping postgresql-k8s-service"
PING postgresql-k8s-service.default.svc.cluster.local (10.102.108.132): 56 data bytes
^C--- postgresql-k8s-service.default.svc.cluster.local ping statistics ---
7 packets transmitted, 0 packets received, 100% packet loss
如果在与 postgresql 相同的节点上运行 Artifactory po，该服务就可以正常工作，这让我相信节点上的 iptables 中出现了问题。
设置：Kubernetes 使用 kubeadm 和 flannel 作为网络提供者。
我尝试了什么？
- 在同一个节点上运行两个 pod 一切正常 gr8 ...
-
在所有节点上运行 iptables -P FORWARD ACCEPT
root@osl-p10y:~# cat /var/run/flannel/subnet.env FLANNEL_NETWORK=10.244.0.0/16 FLANNEL_SUBNET=10.244.1.1/24 法兰绒_MTU=1450 FLANNEL_IPMASQ=true
kubectl get nodes -o jsonpath='{.items[*].spec.podCIDR}' 10.244.2.0/24 10.244.3.0/24 10.244.1.0/24 10.244.0.0/24
在此处输入代码 Postgres 节点。
iptables -t nat -S
-P PREROUTING ACCEPT
-P INPUT ACCEPT
-P OUTPUT ACCEPT
-P POSTROUTING ACCEPT
-N DOCKER
-N KUBE-MARK-DROP
-N KUBE-MARK-MASQ
-N KUBE-NODEPORTS
-N KUBE-POSTROUTING
-N KUBE-SEP-IT2ZTR26TO4XFPTO
-N KUBE-SEP-R6ZMYJ3DNNU76P45
-N KUBE-SEP-SDMS26WNQN2B6OVJ
-N KUBE-SEP-YIL6JZP7A3QYXJU2
-N KUBE-SERVICES
-N KUBE-SVC-6BVLUYEF2BUG3NBU
-N KUBE-SVC-D57225OKWQOKDCSS
-N KUBE-SVC-ERIFXISQEP7F7OF4
-N KUBE-SVC-NPX46M4PTMTKRN6Y
-N KUBE-SVC-TCOU7JCQXEZGVUNU
-A PREROUTING -m comment --comment "kubernetes service portals" -j KUBE-SERVICES
-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
-A OUTPUT -m comment --comment "kubernetes service portals" -j KUBE-SERVICES
-A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
-A POSTROUTING -m comment --comment "kubernetes postrouting rules" -j KUBE-POSTROUTING
-A POSTROUTING -s 172.17.0.0/16 ! -o docker0 -j MASQUERADE
-A POSTROUTING -s 10.244.0.0/16 -d 10.244.0.0/16 -j RETURN
-A POSTROUTING -s 10.244.0.0/16 ! -d 224.0.0.0/4 -j MASQUERADE
-A POSTROUTING ! -s 10.244.0.0/16 -d 10.244.1.0/24 -j RETURN
-A POSTROUTING ! -s 10.244.0.0/16 -d 10.244.0.0/16 -j MASQUERADE
-A DOCKER -i docker0 -j RETURN
-A KUBE-MARK-DROP -j MARK --set-xmark 0x8000/0x8000
-A KUBE-MARK-MASQ -j MARK --set-xmark 0x4000/0x4000
-A KUBE-NODEPORTS -p tcp -m comment --comment "default/postgresql-k8s-service:" -m tcp --dport 30197 -j KUBE-MARK-MASQ
-A KUBE-NODEPORTS -p tcp -m comment --comment "default/postgresql-k8s-service:" -m tcp --dport 30197 -j KUBE-SVC-D57225OKWQOKDCSS
-A KUBE-NODEPORTS -p tcp -m comment --comment "default/artifactory:" -m tcp --dport 30419 -j KUBE-MARK-MASQ
-A KUBE-NODEPORTS -p tcp -m comment --comment "default/artifactory:" -m tcp --dport 30419 -j KUBE-SVC-6BVLUYEF2BUG3NBU
-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -m mark --mark 0x4000/0x4000 -j MASQUERADE
-A KUBE-SEP-IT2ZTR26TO4XFPTO -s 10.244.0.2/32 -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-MARK-MASQ
-A KUBE-SEP-IT2ZTR26TO4XFPTO -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp" -m tcp -j DNAT --to-destination 10.244.0.2:53
-A KUBE-SEP-R6ZMYJ3DNNU76P45 -s 10.5.12.113/32 -m comment --comment "default/kubernetes:https" -j KUBE-MARK-MASQ
-A KUBE-SEP-R6ZMYJ3DNNU76P45 -p tcp -m comment --comment "default/kubernetes:https" -m recent --set --name KUBE-SEP-R6ZMYJ3DNNU76P45 --mask 255.255.255.255 --rsource -m tcp -j DNAT --to-destination 10.5.12.113:6443
-A KUBE-SEP-SDMS26WNQN2B6OVJ -s 172.17.0.2/32 -m comment --comment "default/postgresql-k8s-service:" -j KUBE-MARK-MASQ
-A KUBE-SEP-SDMS26WNQN2B6OVJ -p tcp -m comment --comment "default/postgresql-k8s-service:" -m tcp -j DNAT --to-destination 172.17.0.2:5432
-A KUBE-SEP-YIL6JZP7A3QYXJU2 -s 10.244.0.2/32 -m comment --comment "kube-system/kube-dns:dns" -j KUBE-MARK-MASQ
-A KUBE-SEP-YIL6JZP7A3QYXJU2 -p udp -m comment --comment "kube-system/kube-dns:dns" -m udp -j DNAT --to-destination 10.244.0.2:53
-A KUBE-SERVICES ! -s 10.244.0.0/16 -d 10.96.0.1/32 -p tcp -m comment --comment "default/kubernetes:https cluster IP" -m tcp --dport 443 -j KUBE-MARK-MASQ
-A KUBE-SERVICES -d 10.96.0.1/32 -p tcp -m comment --comment "default/kubernetes:https cluster IP" -m tcp --dport 443 -j KUBE-SVC-NPX46M4PTMTKRN6Y
-A KUBE-SERVICES ! -s 10.244.0.0/16 -d 10.96.0.10/32 -p udp -m comment --comment "kube-system/kube-dns:dns cluster IP" -m udp --dport 53 -j KUBE-MARK-MASQ
-A KUBE-SERVICES -d 10.96.0.10/32 -p udp -m comment --comment "kube-system/kube-dns:dns cluster IP" -m udp --dport 53 -j KUBE-SVC-TCOU7JCQXEZGVUNU
-A KUBE-SERVICES ! -s 10.244.0.0/16 -d 10.96.0.10/32 -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp cluster IP" -m tcp --dport 53 -j KUBE-MARK-MASQ
-A KUBE-SERVICES -d 10.96.0.10/32 -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp cluster IP" -m tcp --dport 53 -j KUBE-SVC-ERIFXISQEP7F7OF4
-A KUBE-SERVICES ! -s 10.244.0.0/16 -d 10.102.108.132/32 -p tcp -m comment --comment "default/postgresql-k8s-service: cluster IP" -m tcp --dport 5432 -j KUBE-MARK-MASQ
-A KUBE-SERVICES -d 10.102.108.132/32 -p tcp -m comment --comment "default/postgresql-k8s-service: cluster IP" -m tcp --dport 5432 -j KUBE-SVC-D57225OKWQOKDCSS
-A KUBE-SERVICES ! -s 10.244.0.0/16 -d 10.101.173.241/32 -p tcp -m comment --comment "default/artifactory: cluster IP" -m tcp --dport 5432 -j KUBE-MARK-MASQ
-A KUBE-SERVICES -d 10.101.173.241/32 -p tcp -m comment --comment "default/artifactory: cluster IP" -m tcp --dport 5432 -j KUBE-SVC-6BVLUYEF2BUG3NBU
-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS
-A KUBE-SVC-D57225OKWQOKDCSS -m comment --comment "default/postgresql-k8s-service:" -j KUBE-SEP-SDMS26WNQN2B6OVJ
-A KUBE-SVC-ERIFXISQEP7F7OF4 -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-SEP-IT2ZTR26TO4XFPTO
-A KUBE-SVC-NPX46M4PTMTKRN6Y -m comment --comment "default/kubernetes:https" -m recent --rcheck --seconds 10800 --reap --name KUBE-SEP-R6ZMYJ3DNNU76P45 --mask 255.255.255.255 --rsource -j KUBE-SEP-R6ZMYJ3DNNU76P45
-A KUBE-SVC-NPX46M4PTMTKRN6Y -m comment --comment "default/kubernetes:https" -j KUBE-SEP-R6ZMYJ3DNNU76P45
-A KUBE-SVC-TCOU7JCQXEZGVUNU -m comment --comment "kube-system/kube-dns:dns" -j KUBE-SEP-YIL6JZP7A3QYXJU2
root@osl-p10y-db:~#
还有来自应用服务器 (Artifactory) 的 iptables nat 规则。
iptables -t nat -vnL | grep -i postgres
5 300 KUBE-MARK-MASQ all -- * * 172.17.0.2 0.0.0.0/0 /* default/postgresql-k8s-service: */
5 300 DNAT tcp -- * * 0.0.0.0/0 0.0.0.0/0 /* default/postgresql-k8s-service: */ tcp to:172.17.0.2:5432
5 300 KUBE-MARK-MASQ tcp -- * * !10.244.0.0/16 10.105.106.161 /* default/postgresql-k8s-service: cluster IP */ tcp dpt:5432
5 300 KUBE-SVC-D57225OKWQOKDCSS tcp -- * * 0.0.0.0/0 10.105.106.161 /* default/postgresql-k8s-service: cluster IP */ tcp dpt:5432
5 300 KUBE-SEP-SDMS26WNQN2B6OVJ all -- * * 0.0.0.0/0 0.0.0.0/0 /* default/postgresql-k8s-service: */
请告知我做错了什么？示例 yaml 在这里：
Arifcatory.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: artifactory-k8s-deployment
spec:
replicas: 1
template:
metadata:
labels:
app: artifactory-pro-k8s
group: artifactory-k8s
spec:
nodeSelector:
name: artfapp2
containers:
- name: artifactory-pro-k8s
image: docker.bintray.io/jfrog/artifactory-pro:5.9.1
env:
- name: DB_TYPE
valueFrom:
configMapKeyRef:
name: k8s-artifactory-db-config
key: DB_TYPE
- name: DB_USER
valueFrom:
secretKeyRef:
name: k8s-artifactory-db-secret
key: POSTGRES_USER
- name: DB_PASSWORD
valueFrom:
secretKeyRef:
name: k8s-artifactory-db-secret
key: POSTGRES_PASSWORD
- name: DB_HOST
valueFrom:
configMapKeyRef:
name: k8s-artifactory-db-config
key: DB_HOST
# Make sure to keep the memory java args aligned with the resources definitions
- name: EXTRA_JAVA_OPTIONS
valueFrom:
configMapKeyRef:
name: k8s-artifactory-config
key: JAVA_OPTS
ports:
- containerPort: 8081
volumeMounts:
- mountPath: "/var/opt/jfrog/artifactory"
name: artifactory-pro-volume
# Make sure to keep the resources set with values matching EXTRA_JAVA_OPTIONS above
resources:
requests:
memory: "1Gi"
cpu: "500m"
limits:
memory: "2Gi"
cpu: "1"
readinessProbe:
httpGet:
path: '/artifactory/webapp/#/login'
port: 8081
initialDelaySeconds: 60
periodSeconds: 10
failureThreshold: 10
livenessProbe:
httpGet:
path: '/artifactory/webapp/#/login'
port: 8081
initialDelaySeconds: 180
periodSeconds: 10
securityContext:
allowPrivilegeEscalation: false
volumes:
- name: artifactory-pro-volume
hostPath:
# directory location on host
path: /srv/data0/artifactory
# this field is optional
type: Directory
---
apiVersion: v1
kind: Service
metadata:
name: artifactory
labels:
app: artifactory
group: artifactory-k8s
spec:
type: NodePort
ports:
- port: 8081
targetPort: 8081
protocol: TCP
selector:
app: artifactory-pro-k8s
Postgresql.yml
kind: Deployment
metadata:
name: postgresql-k8s-deployment
spec:
replicas: 1
template:
metadata:
labels:
app: postgresql-k8s
group: artifactory-k8s
spec:
nodeSelector:
name: artfdb
initContainers:
- name: "remove-lost-found"
image: "busybox:1.26.2"
imagePullPolicy: "IfNotPresent"
command:
- 'sh'
- '-c'
- 'rm -rf /var/lib/postgresql/data/lost+found'
volumeMounts:
- mountPath: "/var/lib/postgresql/data"
name: postgresql-volume
containers:
- name: postgresql-k8s
image: sauce-registry.eng.nutanix.com:5000/nutanix-postgres:latest
env:
- name: POSTGRES_DB
valueFrom:
configMapKeyRef:
name: k8s-artifactory-db-config
key: POSTGRES_DB
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: k8s-artifactory-db-secret
key: POSTGRES_USER
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: k8s-artifactory-db-secret
key: POSTGRES_PASSWORD
ports:
- containerPort: 5432
resources:
requests:
memory: "500Mi"
cpu: "100m"
limits:
memory: "1Gi"
cpu: "500m"
volumeMounts:
- mountPath: "/var/lib/postgresql/data"
name: postgresql-volume
livenessProbe:
exec:
command:
- sh
- -c
- exec pg_isready -U postgres
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 6
readinessProbe:
exec:
command:
- sh
- -c
- exec pg_isready -U postgres
initialDelaySeconds: 30
timeoutSeconds: 3
periodSeconds: 5
volumes:
- name: postgresql-volume
hostPath:
path: /srv/data0/artf_db
type: Directory
---
apiVersion: v1
kind: Service
metadata:
name: postgresql-k8s-service
labels:
app: postgresql-k8s-service
group: artifactory-k8s
spec:
ports:
- port: 5432
protocol: TCP
selector:
app: postgresql-k8s
【问题讨论】：
-
这是来自应用服务器 (Artifactory) 的 iptables nat 规则。
标签： kubernetes