【kubernetes/k8s 部署】minikube 与 kubernetes 搭建

前言

准备工作

1. docker

安装 Docker

分为开源免费的 CE（Community Edition）版本和收费的 EE（Enterprise Edition）版本。

配置 Docker 的 apt 源

允许访问 Docker 源。
$ sudo apt-get install apt-transport-https ca-certificates curl software-properties-common

添加 Docker 官方的 GPG

$ curl -fsSL https://download.docker.com/Linux/ubuntu/gpg | sudo apt-key add -

添加 /etc/apt/sources.list Docker 源

$ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"

安装 Docker

$ sudo apt-get update

$ sudo apt-get docker-ce

2. go

$ wget https://github.com/golang/go/archive/go1.8.3.tar.gz

解压到 /usr/local 目录下，设置环境变量 ~/.profile 添加如下：

export GOROOT=/usr/local/go

export GOPATH=/home/lin/project

export PATH=$GOPATH/bin:$GOROOT/bin:$PATH

3.etcd

下载源码 git clone https://github.com/coreos/etcd.git

编译 ./build

运行 goreman start

4.flanel

下载源码 git clone https://github.com/coreos/flanel.git

编译 make

运行需要权限 # ./flaneld

源码 kubnetes 运行

下载源码 git clone https://github.com/kubernetes/kubernetes.git

编译 make

kube-apiserver

kube-apiserver --etcd-servers=http://127.0.0.1:2379 --service-cluster-ip-range=10.10.10.1/24

/usr/bin/kube-apiserver --logtostderr=true --v=0 --etcd-servers=http://127.0.0.1:2379 --insecure-bind-address=0.0.0.0 --allow-privileged=false --service-cluster-ip-range=10.254.0.0/16 --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,ResourceQuota

# cat apiserver

###
# kubernetes system config
#
# The following values are used to configure the kube-apiserver
#

# The address on the local server to listen to.
KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0"

# The port on the local server to listen on.
# KUBE_API_PORT="--port=8080"

# Port minions listen on
# KUBELET_PORT="--kubelet-port=10250"

# Comma separated list of nodes in the etcd cluster
KUBE_ETCD_SERVERS="--etcd-servers=http://127.0.0.1:2379"

# Address range to use for services
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"

# default admission control policies
#KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,ResourceQuota"

# Add your own!
KUBE_API_ARGS=""

# cat /usr/lib/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
After=etcd.service

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/apiserver
User=kube
ExecStart=/usr/bin/kube-apiserver \
$KUBE_LOGTOSTDERR \
$KUBE_LOG_LEVEL \
$KUBE_ETCD_SERVERS \
$KUBE_API_ADDRESS \
$KUBE_API_PORT \
$KUBELET_PORT \
$KUBE_ALLOW_PRIV \
$KUBE_SERVICE_ADDRESSES \
$KUBE_ADMISSION_CONTROL \
$KUBE_API_ARGS
Restart=on-failure
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

kube-controller-manager

kube-controller-manager --v=0 --master=127.0.0.1:8080

/usr/bin/kube-controller-manager --logtostderr=true --v=0 --master=http://192.168.126.131:8080

# cat controller-manager
###
# The following values are used to configure the kubernetes controller-manager

# defaults from config and apiserver should be adequate

# Add your own!
KUBE_CONTROLLER_MANAGER_ARGS="

# cat /usr/lib/systemd/system/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/GoogleCloudPlatform/kubernetes

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/controller-manager
User=kube
ExecStart=/usr/bin/kube-controller-manager \
$KUBE_LOGTOSTDERR \
$KUBE_LOG_LEVEL \
$KUBE_MASTER \
$KUBE_CONTROLLER_MANAGER_ARGS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

kube-scheduler

kube-scheduler --v=0 --master=127.0.0.1:8080

/usr/bin/kube-scheduler --logtostderr=true --v=0 --master=http://192.168.126.131:8080

# cat scheduler
###
# kubernetes scheduler config

# default config should be adequate

# Add your own!
KUBE_SCHEDULER_ARGS=""

# cat /usr/lib/systemd/system/kube-scheduler.service
[Unit]
Description=Kubernetes Scheduler Plugin
Documentation=https://github.com/GoogleCloudPlatform/kubernetes

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/scheduler
User=kube
ExecStart=/usr/bin/kube-scheduler \
$KUBE_LOGTOSTDERR \
$KUBE_LOG_LEVEL \
$KUBE_MASTER \
$KUBE_SCHEDULER_ARGS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

验证 master 启动

kubectl get componentstatuses

kube-proxy

kube-proxy --v=0 --master=127.0.0.1:8080

/usr/bin/kube-proxy --logtostderr=true --v=0 --master=http://192.168.126.131:8080

cat /etc/kubernetes/proxy
###
# kubernetes proxy config

# default config should be adequate

# Add your own!
KUBE_PROXY_ARGS=""

# cat /usr/lib/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/proxy
ExecStart=/usr/bin/kube-proxy \
$KUBE_LOGTOSTDERR \
$KUBE_LOG_LEVEL \
$KUBE_MASTER \
$KUBE_PROXY_ARGS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

kubelet

kubelet --fail-swap-on=false

/usr/bin/kubelet --logtostderr=true --v=0 --api-servers=http://192.168.126.250:8080 --address=127.0.0.1 --hostname-override=n-192.168.126.250 --allow-privileged=false --pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest

# cat kubelet
###
# kubernetes kubelet (minion) config

# The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
KUBELET_ADDRESS="--address=0.0.0.0"

# The port for the info server to serve on
# KUBELET_PORT="--port=10250"

# You may leave this blank to use the actual hostname
KUBELET_HOSTNAME="--hostname-override=n-192.168.126.130"

# location of the api-server
KUBELET_API_SERVER="--api-servers=http://192.168.126.131:8080"

# pod infrastructure container
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest"

# Add your own!
KUBELET_ARGS=""

# cat /usr/lib/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service

[Service]
WorkingDirectory=/var/lib/kubelet
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/kubelet
ExecStart=/usr/bin/kubelet \
$KUBE_LOGTOSTDERR \
$KUBE_LOG_LEVEL \
$KUBELET_API_SERVER \
$KUBELET_ADDRESS \
$KUBELET_PORT \
$KUBELET_HOSTNAME \
$KUBE_ALLOW_PRIV \
$KUBELET_POD_INFRA_CONTAINER \
$KUBELET_ARGS
Restart=on-failure

[Install]
WantedBy=multi-user.target

验证

创建 nginx 集群

kubectl run nginx --image=nginx --port=100 --replicas=2

kubectl get pods

minikube 安装 kubernetes

下载源码

git clone https://github.com/kubernetes/minikube.git

编译 make

运行 minikube start

dashboard: minikube dashboard

centos7 yum 安装 kubernetes

下载源码

服务器	服务
master	apiserver, controller-manager, scheduler
node	flannel, docker, kubelet, kube-proxy
etcd	etcd

关闭防火墙
systemctl stop firewalld.service
systemctl disable firewalld.service
PS：第一个命令关闭防火墙，第二个命令使开机不启动防火墙（永久关闭）
永久关闭SELinux
修改/etc/selinux/config
SELINUX=disabled
重启机器

安装和配置etcd

3.1 安装etcd

yum install etcd -y

3.2 配置etcd

修改/etc/etcd/etcd.conf中的部分属性
ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379"
ETCD_ADVERTISE_CLIENT_URLS="http://etcd:2379"
PS：其中etcd表示etcd服务器主机名

3.3 运行etcd

systemctl start etcd
systemctl enable etcd

3.4 配置etcd内网信息

etcdctl set /atomic.io/network/config '{"Network":"172.17.0.0/16"}'
PS：atomic.io与下面的Flannel配置中的FLANNEL_ETCD_PREFIX对应

master服务器上安装服务

4.1 安装kubernetes-master

yum install kubernetes-master

4.2 配置apiserver

修改/etc/kubernetes/apiserver
KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0"
KUBE_ETCD_SERVERS="--etcd-servers=http://etcd:2379"
KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,ResourceQuota"
PS：测试时需要把KUBE_ADMISSION_CONTROL中的SecurityContextDeny和ServiceAccount去掉，这是权限相关的

4.3 配置全局配置文件

修改/etc/kubernetes/config
KUBE_MASTER="--master=http://master:8080"

4.4 设置开机启动并启动master服务

systemctl enable kube-apiserver kube-scheduler kube-controller-manager
systemctl start kube-apiserver kube-scheduler kube-controller-manager

4.5 测试master服务

访问http://master:8080会返回json描述信息，如下图：
【kubernetes/k8s 部署】minikube 与 kubernetes 搭建

node节点上安装服务

5.1 安装docker

yum install docker

5.2 安装flannel

yum install flannel

5.3 配置flannel

修改/etc/sysconfig/flanneld
FLANNEL_ETCD_ENDPOINTS="http://etcd:2379"
FLANNEL_ETCD_PREFIX="/atomic.io/network"
PS：其中atomic.io与上面etcd中的Network对应

5.4 安装kubernetes-node

yum install kubernetes-node

5.5 配置全局配置文件

修改/etc/kubernetes/config
KUBE_MASTER="--master=http://master:8080"

5.6 配置kubelet组件

修改/etc/kubernetes/kubelet
KUBELET_HOSTNAME="--hostname-override=node1"
KUBELET_API_SERVER="--api-servers=http://master:8080"

5.7 配置开机启动并启动服务

systemctl enable kubelet kube-proxy
systemctl start kubelet kube-proxy

6.测试集群是否工作

在master节点运行
kubectl get nodes
若正常工作，可获取工作节点信息及运行状态为Ready，如下图：
【kubernetes/k8s 部署】minikube 与 kubernetes 搭建

手动安装 kubernetes

下载源码

git clone https://github.com/kubernetes/kubernetes.git

编译 make

1 配置kube-apiserver

1.1 配置文件

# cat kube-apiserver

# --logtostderr=true: log to standard error instead of files
KUBE_LOGTOSTDERR="--logtostderr=true"

# --v=0: log level for V logs
KUBE_LOG_LEVEL="--v=4"

# --etcd-servers=[]: List of etcd servers to watch (http://ip:port),
# comma separated. Mutually exclusive with -etcd-config
KUBE_ETCD_SERVERS="--etcd-servers=https://127.0.0.1:2379"

# --etcd-cafile="": SSL Certificate Authority file used to secure etcd communication.
#KUBE_ETCD_CAFILE="--etcd-cafile=/srv/kubernetes/etcd/ca.pem"

# --etcd-certfile="": SSL certification file used to secure etcd communication.
KUBE_ETCD_CERTFILE="--etcd-certfile=/srv/kubernetes/etcd/client.pem"

# --etcd-keyfile="": key file used to secure etcd communication.
KUBE_ETCD_KEYFILE="--etcd-keyfile=/srv/kubernetes/etcd/client-key.pem"

# --insecure-bind-address=127.0.0.1: The IP address on which to serve the --insecure-port.
KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0"

# --insecure-port=8080: The port on which to serve unsecured, unauthenticated access.
KUBE_API_PORT="--insecure-port=8080"

# --kubelet-port=10250: Kubelet port
NODE_PORT="--kubelet-port=10250"

# --advertise-address=<nil>: The IP address on which to advertise
# the apiserver to members of the cluster.
KUBE_ADVERTISE_ADDR="--advertise-address=8.8.8.18"

# --allow-privileged=false: If true, allow privileged containers.
KUBE_ALLOW_PRIV="--allow-privileged=false"

# --service-cluster-ip-range=<nil>: A CIDR notation IP range from which to assign service cluster IPs.
# This must not overlap with any IP ranges assigned to nodes for pods.
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"

# --admission-control="AlwaysAdmit": Ordered list of plug-ins
# to do admission control of resources into cluster.
# Comma-delimited list of:
# LimitRanger, AlwaysDeny, SecurityContextDeny, NamespaceExists,
# NamespaceLifecycle, NamespaceAutoProvision, AlwaysAdmit,
# ServiceAccount, DefaultStorageClass, DefaultTolerationSeconds, ResourceQuota
KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,ResourceQuota"

# --client-ca-file="": If set, any request presenting a client certificate signed
# by one of the authorities in the client-ca-file is authenticated with an identity
# corresponding to the CommonName of the client certificate.
KUBE_API_CLIENT_CA_FILE="--client-ca-file=/srv/kubernetes/ca.crt"

# --tls-cert-file="": File containing x509 Certificate for HTTPS. (CA cert, if any,
# concatenated after server cert). If HTTPS serving is enabled, and --tls-cert-file
# and --tls-private-key-file are not provided, a self-signed certificate and key are
# generated for the public address and saved to /var/run/kubernetes.
KUBE_API_TLS_CERT_FILE="--tls-cert-file=/srv/kubernetes/server.cert"

# --tls-private-key-file="": File containing x509 private key matching --tls-cert-file.
KUBE_API_TLS_PRIVATE_KEY_FILE="--tls-private-key-file=/srv/kubernetes/server.key"

1.2 配置启动脚本

# cat /usr/lib/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-apiserver
ExecStart=/opt/kubernetes/bin/kube-apiserver ${KUBE_LOGTOSTDERR} \
${KUBE_LOG_LEVEL} \
${KUBE_ETCD_SERVERS} \
${KUBE_API_ADDRESS} \
${KUBE_API_PORT} \
${NODE_PORT} \
${KUBE_ALLOW_PRIV} \
&{KUBE_SERVICE_ADDRESSES}
Restart=on-failure

[Install]
WantedBy=multi-user.target

2 配置kube-controller-manager

2.1 配置文件

# cat kube-controller-manager
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=4"
KUBE_MASTER="--master=127.0.0.1:8080"

# --root-ca-file="": If set, this root certificate authority will be included in
# service account's token secret. This must be a valid PEM-encoded CA bundle.
KUBE_CONTROLLER_MANAGER_ROOT_CA_FILE="--root-ca-file=/srv/kubernetes/ca.crt"

# --service-account-private-key-file="": Filename containing a PEM-encoded private
# RSA key used to sign service account tokens.
KUBE_CONTROLLER_MANAGER_SERVICE_ACCOUNT_PRIVATE_KEY_FILE="--service-account-private-key-file=/srv/kubernetes/server.key"

# --leader-elect
KUBE_LEADER_ELECT="--leader-elect"

2.2 启动脚本文件

[[email protected] cfg]# cat /usr/lib/systemd/system/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-controller-manager
ExecStart=/opt/kubernetes/bin/kube-controller-manager ${KUBE_LOGTOSTDERR} \
${KUBE_LOG_LEVEL} \
${KUBE_MASTER}
Restart=on-failure

[Install]
WantedBy=multi-user.target

3 配置kube-scheduler

1.1 配置文件

# cat kube-scheduler

###
# kubernetes scheduler config

# --logtostderr=true: log to standard error instead of files
KUBE_LOGTOSTDERR="--logtostderr=true"

# --v=0: log level for V logs
KUBE_LOG_LEVEL="--v=4"

KUBE_MASTER="--master=127.0.0.1:8080"

# --leader-elect
KUBE_LEADER_ELECT="--leader-elect"

# Add your own!
KUBE_SCHEDULER_ARGS=""

3.2 启动脚本文件

# cat /usr/lib/systemd/system/kube-scheduler.service

[Unit]

Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-scheduler
ExecStart=/opt/kubernetes/bin/kube-scheduler ${KUBE_LOGTOSTDERR} \
${KUBE_LOG_LEVEL} \
${KUBE_MASTER} \
$KUBE_SCHEDULER_ARGS
Restart=on-failure
[Install]
WantedBy=multi-user.target

4 配置kube-proxy

4.1 配置文件

# cat kube-proxy

# --logtostderr=true: log to standard error instead of files

KUBE_LOGTOSTDERR="--logtostderr=true"

# --v=0: log level for V logs
KUBE_LOG_LEVEL="--v=4"

# --hostname-override="": If non-empty, will use this string as identification instead of the actual hostname.
NODE_HOSTNAME="--hostname-override=127.0.0.1"

# --master="": The address of the Kubernetes API server (overrides any value in kubeconfig)
KUBE_MASTER="--master=http://127.0.0.1:8080"

4.2 启动脚本文件

# cat /usr/lib/systemd/system/kube-proxy.service

[Unit]
Description=Kubernetes Proxy
After=network.target

[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-proxy
ExecStart=/opt/kubernetes/bin/kube-proxy ${KUBE_LOGTOSTDERR} \
${KUBE_LOG_LEVEL} \
${NODE_HOSTNAME} \
${KUBE_MASTER}
Restart=on-failure

[Install]
WantedBy=multi-user.target

5 配置kubelet

5.1 配置文件

# cat kubelet

# --logtostderr=true: log to standard error instead of files
KUBE_LOGTOSTDERR="--logtostderr=true"

# --v=0: log level for V logs
KUBE_LOG_LEVEL="--v=4"

# --address=0.0.0.0: The IP address for the Kubelet to serve on (set to 0.0.0.0 for all interfaces)
NODE_ADDRESS="--address=0.0.0.0"

# --port=10250: The port for the Kubelet to serve on. Note that "kubectl logs" will not work if you set this flag.
NODE_PORT="--port=10250"

# --hostname-override="": If non-empty, will use this string as identification instead of the actual hostname.
NODE_HOSTNAME="--hostname-override=local-node"

# Path to a kubeconfig file, specifying how to connect to the API server.
KUBELET_KUBECONFIG="--kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig"

# --allow-privileged=false: If true, allow containers to request privileged mode. [default=false]
KUBE_ALLOW_PRIV="--allow-privileged=false"

# DNS info
KUBELET__DNS_IP="--cluster-dns=192.168.3.100"
KUBELET_DNS_DOMAIN="--cluster-domain=cluster.local"

# Add your own!
KUBELET_ARGS=""

5.2 启动脚本文件

# cat /usr/lib/systemd/system/kubelet.service

[Unit]
Description=Kubernetes Kubelet
After=docker.service
Requires=docker.service

[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kubelet
ExecStart=/opt/kubernetes/bin/kubelet ${KUBE_LOGTOSTDERR} \
${KUBE_LOG_LEVEL} \
${NODE_ADDRESS} \
${NODE_PORT} \
${NODE_HOSTNAME} \
${KUBELET_KUBECONFIG} \
${KUBE_ALLOW_PRIV} \
${KUBELET__DNS_IP} \
--fail-swap-on=false --pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest
Restart=on-failure
KillMode=process

[Install]
WantedBy=multi-user.target

安装 kubernetes dashboard

dashboard.yaml

kind: Deployment
apiVersion: extensions/v1beta1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: kubernetes-dashboard
template:
metadata:
labels:
k8s-app: kubernetes-dashboard
# Comment the following annotation if Dashboard must not be deployed on master
annotations:
scheduler.alpha.kubernetes.io/tolerations: |
[
{
"key": "dedicated",
"operator": "Equal",
"value": "master",
"effect": "NoSchedule"
}
]
spec:
containers:
- name: kubernetes-dashboard
image: mritd/kubernetes-dashboard-amd64
ports:
- containerPort: 9090
protocol: TCP
args:
# Uncomment the following line to manually specify Kubernetes API server Host
# If not specified, Dashboard will attempt to auto discover the API server and connect
# to it. Uncomment only if the default does not work.
- --apiserver-host=http://172.23.150.97:8080
livenessProbe:
httpGet:
path: /
port: 9090
initialDelaySeconds: 30
timeoutSeconds: 30
---
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
spec:
ports:
- port: 80
targetPort: 9090
selector:
k8s-app: kubernetes-dashboard

kubectl create -f dashboard.yaml

kubectl proxy

kubectl get services --all-namespaces