如何将审核日志发送到 minikube 中的 webhook

Question

在 minikube 中，API 服务器无法连接到我的审核日志 webhook，我在 api-server 日志中看到以下错误

E0308 08:30:26.457841       1 metrics.go:109] Error in audit plugin 'webhook' affecting 400 audit events: Post "http://ca-audit.armo-system:8888/": dial tcp: lookup ca-audit.armo-system on 10.42.4.254:53: no such host

我不知道为什么 api-server 连接到 10.42.4.254:53，因为我的服务 ip 不同：

$ kubectl -n armo-system get services
NAME       TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE
ca-audit   ClusterIP   10.109.132.114   <none>        8888/TCP   8m33s

我不明白我做错了什么，有什么建议吗？

---

这就是我配置审计策略、webhook 和 minikube 的方式-

我预先配置了我的 minikube 如下：

# Create the webhook config and audit policy
export C_CONTEXT=$(kubectl config current-context)
export C_NAME=$(kubectl config get-contexts ${C_CONTEXT} --no-headers | awk '{print $2}')
export C_CLUSTER=$(kubectl config get-contexts ${C_CONTEXT} --no-headers | awk '{print $3}')
export C_USER=$(kubectl config get-contexts ${C_CONTEXT} --no-headers | awk '{print $4}')

export ARMO_NAMESPACE="armo-system"
export ARMO_AUDIT_SERVICE="ca-audit"
export ARMO_AUDIT_PORT=8888

mkdir -p ~/.minikube/files/etc/ssl/certs

cat <<EOF > ~/.minikube/files/etc/ssl/certs/audit-webhook.yaml
{
  "apiVersion": "v1",
  "clusters": [
    {
      "cluster": {
        "server": "http://${ARMO_AUDIT_SERVICE}.${ARMO_NAMESPACE}:${ARMO_AUDIT_PORT}/"
      },
      "name": "${C_NAME}"
    }
  ],
  "contexts": [
    {
      "context": {
        "cluster": "${C_CLUSTER}",
        "user": "${C_USER}"
      },
      "name": "${C_NAME}"
    }
  ],
  "current-context": "${C_CONTEXT}",
  "kind": "Config",
  "preferences": {},
  "users": []
}
EOF

cat <<EOF > ~/.minikube/files/etc/ssl/certs/audit-policy.yaml
{
  "apiVersion": "audit.k8s.io/v1",
  "kind": "Policy",
  "rules": [
    {
      "level": "Metadata"
    }
  ]
}
EOF

# Copy the audit policy to `/etc/ssl/certs/.` 
sudo cp ~/.minikube/files/etc/ssl/certs/audit-policy.yaml ~/.minikube/files/etc/ssl/certs/audit-webhook.yaml /etc/ssl/certs/.

# Start the minikube, add the flags `--extra-config=apiserver.audit-policy-file=/etc/ssl/certs/audit-policy.yaml`, `--extra-config=apiserver.audit-webhook-config-file=/etc/ssl/certs/audit-webhook.yaml`
sudo -E minikube start --vm-driver=none --extra-config=apiserver.audit-policy-file=/etc/ssl/certs/audit-policy.yaml --extra-config=apiserver.audit-webhook-config-file=/etc/ssl/certs/audit-webhook.yaml

现在我的 minikube 已经启动并运行了，我创建了命名空间、服务和 webhook 部署：

cat <<EOF | kubectl apply -f -
---
apiVersion: v1
kind: Namespace
metadata:
  name: ${ARMO_NAMESPACE}

---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: ${ARMO_AUDIT_SERVICE}
  name: ${ARMO_AUDIT_SERVICE}
  namespace: ${ARMO_NAMESPACE}
spec:
  ports:
  - port: ${ARMO_AUDIT_PORT}
    targetPort: ${ARMO_AUDIT_PORT}
    protocol: TCP
  selector:
    app: ${ARMO_AUDIT_SERVICE}
---
apiVersion: apps/v1
kind: Deployment
metadata:
    name: ${ARMO_AUDIT_SERVICE}
    namespace: ${ARMO_NAMESPACE}
    labels:
      app: ${ARMO_AUDIT_SERVICE}
spec:
  selector:
    matchLabels:
      app: ${ARMO_AUDIT_SERVICE}
  replicas: 1
  template:
    metadata:
      labels:
        app: ${ARMO_AUDIT_SERVICE}
    spec:
      containers:
        - name: ${ARMO_AUDIT_SERVICE}
          image: quay.io/armosec/k8s-ca-auditlog-ubi:dummy
          imagePullPolicy: Always
          env:
          - name: ARMO_AUDIT_PORT
            value: "${ARMO_AUDIT_PORT}"
          ports:
          - containerPort: ${ARMO_AUDIT_PORT}
            name: ${ARMO_AUDIT_SERVICE}
EOF

webhook 图片代码（quay.io/armosec/k8s-ca-auditlog-ubi:dummy）如下：

package main

import (
  "encoding/json"
  "flag"
  "fmt"
  "net/http"
  "os"

  "k8s.io/apiserver/pkg/apis/audit"

  "github.com/golang/glog"
)

func main() {
  flag.Parse()
  flag.Set("alsologtostderr", "1") // display logs in stdout

  InitServer()

}

// InitServer - Initialize webhook listener
func InitServer() {
  port, ok := os.LookupEnv("ARMO_AUDIT_PORT")
  if !ok {
    port = "8888"
  }
  glog.Infof("Webhook listening on port: %s, path: %s", port, "/")
  http.HandleFunc("/", HandleRequest)
  glog.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil))
}

//HandleRequest -
func HandleRequest(w http.ResponseWriter, req *http.Request) {
  eventList := audit.EventList{}
  err := json.NewDecoder(req.Body).Decode(&eventList)
  if err != nil {
    e := fmt.Errorf("failed parsing api-server request, reason: %s", err.Error())
    glog.Errorf(e.Error())
    http.Error(w, e.Error(), http.StatusBadRequest)
    return
  }
  glog.Infof("webhook received audit list, len: %d", len(eventList.Items))
  for _, event := range eventList.Items {
    bEvent, _ := json.Marshal(event)
    glog.Infof("Received event: %s", string(bEvent))
  }

  w.WriteHeader(http.StatusOK)
}

Answer 1

其实我对 minikube 不太了解，但我使用了 k8s 的审计，所以我的回答对 minikube 环境没有帮助。

首先，k8s dns 格式不正确。它应该是 {service-name}.{namespace}.svc 或 {service-name}.{namespace}.svc.cluster.local。当然也可以加上http://或者https://。

第二，你应该给dnsPolicy选项kube-apiserver.yaml。 kube-apiserver在集群中搜索dns解析，需要写‘dnsPolicy’。

sudo vi /etc/kubernetes/manifests/kube-apiserver.yaml

打开 kube-apiserver.yaml 并将 dnsPolicy: ClusterFirstWithHostNet 添加到其中。 dnsPolicy 的位置低于 .spec 并且与容器或卷处于同一级别