#参考文档：https://github.com/opsnull/follow-me-install-kubernetes-cluster

#约定：

#1、会显式的标注出命令都要再哪一台或哪几台上执行；

#2、本次不配置高可用apiserver

#3、ETCD集群配置不加密模式；

#4、安装镜像使用默认的镜像，被墙掉的镜像需要提前导入到系统中，可以参考文章末尾的相关操作导入；

#5、Master主机 172.16.3.150, node主机 172.16.3.151,172.16.3.152, 测试扩充|删除节点主机 172.16.3.153

#######################-----初始化系统-----#######################

#此部分的命令需要三台服务器都执行

#初始化主机列表:172.16.3.150,172.16.3.151,172.16.3.152

#修改主机名

hostnamectl set-hostname dev-test-3-150 #172.16.3.150执行

hostnamectl set-hostname dev-test-3-151 #172.16.3.151执行

hostnamectl set-hostname dev-test-3-152 #172.16.3.162执行

#yum升级及相关组件安装

yum install epel-release -y

yum update -y

yum install lrzsz vim wget net-tools ntp python-pip conntrack ipvsadm ipset jq sysstat curl iptables libseccomp conntrack-tools -y

yum install -y bash-completion

pip install --upgrade pip

#优化系统参数()

echo "net.ipv4.tcp_fin_timeout=30">>/etc/sysctl.conf

echo "net.ipv4.tcp_tw_recycle=1">>/etc/sysctl.conf

echo "net.ipv4.tcp_tw_reuse=1">>/etc/sysctl.conf

echo "net.ipv4.icmp_echo_ignore_broadcasts=1">>/etc/sysctl.conf

echo "net.ipv4.conf.all.rp_filter=1">>/etc/sysctl.conf

echo "net.ipv4.tcp_keepalive_time=300">>/etc/sysctl.conf

echo "net.ipv4.tcp_synack_retries=2">>/etc/sysctl.conf

echo "net.ipv4.tcp_syn_retries=2">>/etc/sysctl.conf

echo "net.ipv4.ip_forward=1">>/etc/sysctl.conf

sysctl -p

echo "*softnofile=65536">>/etc/security/limits.conf

echo "*hardnofile=65536">>/etc/security/limits.conf

echo "ulimit -n 65536">>/etc/profile

source /etc/profile

#ntp配置

service ntpd start

chkconfig ntpd on

#iptables配置

systemctl stop firewalld

systemctl disable firewalld

yum install -y iptables-services iptables-devel.x86_64 iptables.x86_64 

service iptables start 

chkconfig iptables on

iptables -F &&  iptables -X &&  iptables -F -t nat &&  iptables -X -t nat

iptables -P FORWARD ACCEPT

#增加hosts

vim /etc/hosts

#添加如下内容

172.16.3.150 dev-test-3-150

172.16.3.151 dev-test-3-151

172.16.3.152 dev-test-3-152

#关闭 swap 分区

swapoff -a

#为了防止开机自动挂载 swap 分区，可以注释 /etc/fstab 中相应的条目

sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

#无密码 ssh 登录其它节点

#生成密钥

ssh-keygen -t rsa

#拷贝密钥

ssh-copy-id root@dev-test-3-150

ssh-copy-id root@dev-test-3-151

ssh-copy-id root@dev-test-3-152

#创建相关安装目录

mkdir -p /opt/k8s/bin

mkdir -p /etc/kubernetes/cert

#mkdir -p /etc/etcd/cert

mkdir -p /var/lib/etcd

#新建环境变量文件,根据实际主机修改NODE_IP和NODE_NAME

vim /opt/k8s/env.sh

# 生成 EncryptionConfig 所需的加密 key

export ENCRYPTION_KEY=$(head -c 32 /dev/urandom | base64)

# TLS Bootstrapping 使用的 Token，可以使用命令 head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 生成

#export BOOTSTRAP_TOKEN="261e41eb026ccacc5d71c7698392c00e"

# 建议用 未用的网段 来定义服务网段和 Pod 网段

# 服务网段 (Service CIDR），部署前路由不可达，部署后集群内使用 IP:Port可达

export SERVICE_CIDR="10.253.0.0/16"

# POD 网段 (Cluster CIDR），部署前路由不可达，**部署后**路由可达 (flanneld 保证)

export CLUSTER_CIDR="172.50.0.0/16"

# 服务端口范围 (NodePort Range)

export NODE_PORT_RANGE="10000-30000"

# etcd 集群服务地址列表

export ETCD_ENDPOINTS="http://172.16.3.150:2379,http://172.16.3.151:2379,http://172.16.3.152:2379"

# flanneld 网络配置前缀

export FLANNEL_ETCD_PREFIX="/kubernetes/network"

# kubernetes 服务 IP (预分配，一般是 SERVICE_CIDR 中第一个IP)

export CLUSTER_KUBERNETES_SVC_IP="10.253.0.1"

# 集群 DNS 服务 IP (从 SERVICE_CIDR 中预分配)

export CLUSTER_DNS_SVC_IP="10.253.0.2"

# 集群 DNS 域名

export CLUSTER_DNS_DOMAIN="cluster.local."

# 部署机器的主机名(修改为对应主机)

export NODE_NAME=dev-test-3-150

# 当前部署主机的IP(修改为对应主机)

export NODE_IP=172.16.3.150

# ETCD集群所有主机的ip地址

export NODE_IPS="172.16.3.150 172.16.3.151 172.16.3.152"

# ETCD集群所有主机的主机名

export NODE_NAMES="dev-test-3-150 dev-test-3-151 dev-test-3-152"

# ETCD集群通讯的ip和端口

export ETCD_NODES=dev-test-3-150=http://172.16.3.150:2380,dev-test-3-151=http://172.16.3.151:2380,dev-test-3-152=http://172.16.3.152:2380

# 临时的Kubernetes的api接口地址

export KUBE_APISERVER="https://172.16.3.150:6443"

#Master API Server 地址

export MASTER_URL="k8s-api.virtual.local"

#Master IP 地址

export MASTER_IP="172.16.3.150"

#二进制文件目录

export PATH=/opt/k8s/bin:$PATH

#保存退出

#关闭selinux

vim /etc/selinux/config

SELINUX=enforcing  改成 :SELINUX=disabled

#保存退出

#重启服务器

init 6

#######################-----创建 CA 证书和秘钥-----#######################

#CA 证书是集群所有节点共享的，只需要创建一个 CA 证书，后续创建的所有证书都由它签名。

#安装CFSSL(三个节点都装)

mkdir /root/ssl/ && cd /root/ssl

wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64

wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64

wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64

chmod u+x *linux-amd64

mv cfssl_linux-amd64 /usr/bin/cfssl

mv cfssljson_linux-amd64 /usr/bin/cfssljson

mv cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo

#配置CA证书(只在3.150上配置)

#创建配置文件

#CA 配置文件用于配置根证书的使用场景 (profile) 和具体参数 (usage，过期时间、服务端认证、客户端认证、加密等)，后续在签名其它证书时需要指定特定场景。

cd /home

source /opt/k8s/env.sh

cfssl print-defaults config > config.json

cfssl print-defaults csr > csr.json

cat > ca-config.json <<EOF

{

  "signing": {

    "default": {

      "expiry": "87600h"

    },

    "profiles": {

      "kubernetes": {

        "usages": [

            "signing",

            "key encipherment",

            "server auth",

            "client auth"

        ],

        "expiry": "87600h"

      }

    }

  }

}

EOF

#signing：表示该证书可用于签名其它证书，生成的 ca.pem 证书中 CA=TRUE；

#server auth：表示 client 可以用该该证书对 server 提供的证书进行验证；

#client auth：表示 server 可以用该该证书对 client 提供的证书进行验证；

#创建证书签名请求文件

cat > ca-csr.json <<EOF

{

  "CN": "kubernetes",

  "key": {

    "algo": "rsa",

    "size": 2048

  },

  "names": [

    {

      "C": "CN",

      "ST": "BeiJing",

      "L": "BeiJing",

      "O": "k8s",

      "OU": "QRXD"

    }

  ]

}

EOF

#CN：Common Name，kube-apiserver 从证书中提取该字段作为请求的用户名 (User Name)，浏览器使用该字段验证网站是否合法；

#O：Organization，kube-apiserver 从证书中提取该字段作为请求用户所属的组 (Group)；

#kube-apiserver 将提取的 User、Group 作为 RBAC 授权的用户标识；

#生成 CA 证书和私钥(只在3.150上配置)

cfssl gencert -initca ca-csr.json | cfssljson -bare ca

ls ca*

scp ca*.pem ca-config.json  root@172.16.3.150:/etc/kubernetes/cert/

scp ca*.pem ca-config.json  root@172.16.3.151:/etc/kubernetes/cert/

scp ca*.pem ca-config.json  root@172.16.3.152:/etc/kubernetes/cert/

#######################-----部署 kubectl 命令行工具-----#######################

#只需要部署一次(只在3.150部署配置)

#下载

cd /home

wget https://dl.k8s.io/v1.10.4/kubernetes-server-linux-amd64.tar.gz

tar -zxvf kubernetes-server-linux-amd64.tar.gz

scp /home/kubernetes/server/bin/kubectl root@172.16.3.150:/usr/bin/

scp /home/kubernetes/server/bin/kubectl root@172.16.3.151:/usr/bin/

scp /home/kubernetes/server/bin/kubectl root@172.16.3.152:/usr/bin/

#创建 admin 证书和私钥(只在3.150上配置)

#kubectl 与 apiserver https 安全端口通信，apiserver 对提供的证书进行认证和授权。

#kubectl 作为集群的管理工具，需要被授予最高权限。这里创建具有最高权限的 admin 证书。

cd /home

cat > admin-csr.json <<EOF

{

  "CN": "admin",

  "hosts": [],

  "key": {

    "algo": "rsa",

    "size": 2048

  },

  "names": [

    {

      "C": "CN",

      "ST": "BeiJing",

      "L": "BeiJing",

      "O": "system:masters",

      "OU": "QRXD"

    }

  ]

}

EOF

#O 为 system:masters，kube-apiserver 收到该证书后将请求的 Group 设置为 system:masters；

#预定义的 ClusterRoleBinding cluster-admin 将 Group system:masters 与 Role cluster-admin 绑定，该 Role 授予所有 API的权限；

#该证书只会被 kubectl 当做 client 证书使用，所以 hosts 字段为空；

#生成证书和私钥

cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \

  -ca-key=/etc/kubernetes/cert/ca-key.pem \

  -config=/etc/kubernetes/cert/ca-config.json \

  -profile=kubernetes admin-csr.json | cfssljson -bare admin

ls admin*

#创建 kubeconfig 文件

#kubeconfig 为 kubectl 的配置文件，包含访问 apiserver 的所有信息，如 apiserver 地址、CA 证书和自身使用的证书；

source /opt/k8s/env.sh

# 设置集群参数

kubectl config set-cluster kubernetes \

  --certificate-authority=/etc/kubernetes/cert/ca.pem \

  --embed-certs=true \

  --server=${KUBE_APISERVER} \

  --kubeconfig=kubectl.kubeconfig

# 设置客户端认证参数

kubectl config set-credentials admin \

  --client-certificate=admin.pem \

  --client-key=admin-key.pem \

  --embed-certs=true \

  --kubeconfig=kubectl.kubeconfig

# 设置上下文参数

kubectl config set-context kubernetes \

  --cluster=kubernetes \

  --user=admin \

  --kubeconfig=kubectl.kubeconfig

# 设置默认上下文

kubectl config use-context kubernetes --kubeconfig=kubectl.kubeconfig

#--certificate-authority：验证 kube-apiserver 证书的根证书；

#--client-certificate、--client-key：刚生成的 admin 证书和私钥，连接 kube-apiserver 时使用；

#--embed-certs=true：将 ca.pem 和 admin.pem 证书内容嵌入到生成的 kubectl.kubeconfig 文件中(不加时，写入的是证书文件路径)；

mkdir /root/.kube

scp kubectl.kubeconfig root@172.16.3.150:/root/.kube/config

scp kubectl.kubeconfig root@172.16.3.151:/root/.kube/config

scp kubectl.kubeconfig root@172.16.3.152:/root/.kube/config

#scp kubectl.kubeconfig root@172.16.3.153:/root/.kube/config

#配置kubectl tab命令

source /usr/share/bash-completion/bash_completion

source <(kubectl completion bash)

#echo "source <(kubectl completion bash)" >> ~/.bashrc

#

#######################-----部署 etcd 集群(非加密)-----#######################

#三个节点都执行

cd /home

wget https://github.com/coreos/etcd/releases/download/v3.1.19/etcd-v3.1.19-linux-amd64.tar.gz

tar -zxvf etcd-v3.1.19-linux-amd64.tar.gz

cp etcd-v3.1.19-linux-amd64/etcd* /usr/local/bin

cd /etc/systemd/system/

source /opt/k8s/env.sh

cat > etcd.service <<EOF

[Unit]

Description=Etcd Server

After=network.target

After=network-online.target

Wants=network-online.target

Documentation=https://github.com/coreos

[Service]

Type=notify

WorkingDirectory=/var/lib/etcd/

ExecStart=/usr/local/bin/etcd \\

  --name=${NODE_NAME} \\

  --initial-advertise-peer-urls=http://${NODE_IP}:2380 \\

  --listen-peer-urls=http://${NODE_IP}:2380 \\

  --listen-client-urls=http://${NODE_IP}:2379,http://127.0.0.1:2379 \\

  --advertise-client-urls=http://${NODE_IP}:2379 \\

  --initial-cluster-token=etcd-cluster-0 \\

  --initial-cluster=${ETCD_NODES} \\

  --initial-cluster-state=new \\

  --data-dir=/var/lib/etcd

Restart=on-failure

RestartSec=5

LimitNOFILE=65536

[Install]

WantedBy=multi-user.target

EOF

#如果提示访问端口不通，防火墙需要开放2379和2380端口

#iptables -A INPUT -p tcp -m state --state NEW -m tcp --dport 2379 -j ACCEPT

#iptables -A INPUT -p tcp -m state --state NEW -m tcp --dport 2380 -j ACCEPT

#service iptables save

systemctl daemon-reload

systemctl enable etcd

systemctl restart etcd

etcdctl --endpoints=${ETCD_ENDPOINTS} cluster-health

#member 224d21bc6c7cdd08 is healthy: got healthy result from http://172.16.3.151:2379

#member 254a600a75e4f39f is healthy: got healthy result from http://172.16.3.152:2379

#member b6f4f1b2c016095e is healthy: got healthy result from http://172.16.3.150:2379

#cluster is healthy

#######################-----部署 flannel 网络-----#######################

#kubernetes 要求集群内各节点(包括 master 节点)能通过 Pod 网段互联互通。flannel 使用 vxlan 技术为各节点创建一个可以互通的 Pod 网络。

#flaneel 第一次启动时，从 etcd 获取 Pod 网段信息，为本节点分配一个未使用的 /24 段地址，然后创建 flannedl.1（也可能是其它名称，如 flannel1 等） 接口。

#flannel 将分配的 Pod 网段信息写入 /run/flannel/docker 文件，docker 后续使用这个文件中的环境变量设置 docker0 网桥。

#三台服务器都执行

cd /home

wget https://github.com/coreos/flannel/releases/download/v0.9.0/flannel-v0.9.0-linux-amd64.tar.gz

tar -zxvf flannel-v0.9.0-linux-amd64.tar.gz

cp flanneld /usr/local/bin/

cp mk-docker-opts.sh /usr/local/bin/

mkdir -p /etc/flanneld/cert

#创建 flannel 证书和私钥(3.150执行)

cd /home

cat > flanneld-csr.json <<EOF

{

  "CN": "flanneld",

  "hosts": [],

  "key": {

    "algo": "rsa",

    "size": 2048

  },

  "names": [

    {

      "C": "CN",

      "ST": "BeiJing",

      "L": "BeiJing",

      "O": "k8s",

      "OU": "QRXD"

    }

  ]

}

EOF

#该证书只会被 kubectl 当做 client 证书使用，所以 hosts 字段为空；

cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \

  -ca-key=/etc/kubernetes/cert/ca-key.pem \

  -config=/etc/kubernetes/cert/ca-config.json \

  -profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld

ls flanneld*pem

scp flanneld*pem root@172.16.3.150:/etc/flanneld/cert/

scp flanneld*pem root@172.16.3.151:/etc/flanneld/cert/

scp flanneld*pem root@172.16.3.152:/etc/flanneld/cert/

#向 etcd 写入集群 Pod 网段信息(3.150执行)

source /opt/k8s/env.sh

etcdctl --endpoints=${ETCD_ENDPOINTS} set ${FLANNEL_ETCD_PREFIX}/config '{"Network":"'${CLUSTER_CIDR}'", "SubnetLen": 24, "Backend": {"Type": "vxlan"}}'

#测试

etcdctl --endpoints=${ETCD_ENDPOINTS} get ${FLANNEL_ETCD_PREFIX}/config

#创建flanneld的systemd unit 文件(3个节点执行)

source /opt/k8s/env.sh

cd /etc/systemd/system/

cat > flanneld.service << EOF

[Unit]

Description=Flanneld overlay address etcd agent

After=network.target

After=network-online.target

Wants=network-online.target

After=etcd.service

Before=docker.service

[Service]

Type=notify

ExecStart=/usr/local/bin/flanneld \\

  -etcd-cafile=/etc/kubernetes/cert/ca.pem \\

  -etcd-certfile=/etc/flanneld/cert/flanneld.pem \\

  -etcd-keyfile=/etc/flanneld/cert/flanneld-key.pem \\

  -etcd-endpoints=${ETCD_ENDPOINTS} \\

  -etcd-prefix=${FLANNEL_ETCD_PREFIX}

ExecStartPost=/usr/local/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker

Restart=on-failure

[Install]

WantedBy=multi-user.target

RequiredBy=docker.service

EOF

#启动flanneld

systemctl daemon-reload

systemctl enable flanneld

systemctl start flanneld

#检查

etcdctl --endpoints=${ETCD_ENDPOINTS} ls ${FLANNEL_ETCD_PREFIX}/subnets

#######################-----部署 master 节点-----#######################

#在3.150上部署

cd /home

#wget https://dl.k8s.io/v1.10.4/kubernetes-server-linux-amd64.tar.gz

#tar -zxvf kubernetes-server-linux-amd64.tar.gz

cd kubernetes

tar -xzvf  kubernetes-src.tar.gz

cp server/bin/{kube-apiserver,kube-controller-manager,kube-scheduler} /usr/local/bin/ 

#部署 kube-apiserver 组件

#创建 kubernetes 证书和私钥

cd /home

source /opt/k8s/env.sh

cat > kubernetes-csr.json <<EOF

{

  "CN": "kubernetes",

  "hosts": [

    "127.0.0.1",

    "172.16.3.150",

    "172.16.3.151",

    "172.16.3.152",

    "172.16.0.0/16",

    "${CLUSTER_KUBERNETES_SVC_IP}",

    "kubernetes",

    "kubernetes.default",

    "kubernetes.default.svc",

    "kubernetes.default.svc.cluster",

    "kubernetes.default.svc.cluster.local"

  ],

  "key": {

    "algo": "rsa",

    "size": 2048

  },

  "names": [

    {

      "C": "CN",

      "ST": "BeiJing",

      "L": "BeiJing",

      "O": "k8s",

      "OU": "QRXD"

    }

  ]

}

EOF

cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \

  -ca-key=/etc/kubernetes/cert/ca-key.pem \

  -config=/etc/kubernetes/cert/ca-config.json \

  -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes

ls kubernetes*pem

cp kubernetes*pem /etc/kubernetes/cert

#创建加密配置文件

# cat > token.csv <<EOF

# ${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"

# EOF

# mv token.csv /etc/kubernetes/

source /opt/k8s/env.sh

cat > encryption-config.yaml <<EOF

kind: EncryptionConfig

apiVersion: v1

resources:

  - resources:

      - secrets

    providers:

      - aescbc:

          keys:

            - name: key1

              secret: ${ENCRYPTION_KEY}

      - identity: {}

EOF

cp encryption-config.yaml /etc/kubernetes/encryption-config.yaml 

#创建kube-apiserver 的systemd unit文件

cd /etc/systemd/system

mkdir /var/log/kube

source /opt/k8s/env.sh

cat  > kube-apiserver.service <<EOF

[Unit]

Description=Kubernetes API Server

Documentation=https://github.com/GoogleCloudPlatform/kubernetes

After=network.target

[Service]

ExecStart=/usr/local/bin/kube-apiserver \\

  --enable-admission-plugins=Initializers,NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \\

  --anonymous-auth=false \\

  --experimental-encryption-provider-config=/etc/kubernetes/encryption-config.yaml \\

  --insecure-bind-address=${MASTER_IP} \\

  --advertise-address=${NODE_IP} \\

  --bind-address=${MASTER_IP} \\

  --authorization-mode=Node,RBAC \\

  --runtime-config=api/all \\

  --enable-bootstrap-token-auth \\

  --service-cluster-ip-range=${SERVICE_CIDR} \\

  --service-node-port-range=${NODE_PORT_RANGE} \\

  --tls-cert-file=/etc/kubernetes/cert/kubernetes.pem \\

  --tls-private-key-file=/etc/kubernetes/cert/kubernetes-key.pem \\

  --client-ca-file=/etc/kubernetes/cert/ca.pem \\

  --kubelet-client-certificate=/etc/kubernetes/cert/kubernetes.pem \\

  --kubelet-client-key=/etc/kubernetes/cert/kubernetes-key.pem \\

  --service-account-key-file=/etc/kubernetes/cert/ca-key.pem \\

  --etcd-servers=${ETCD_ENDPOINTS} \\

  --enable-swagger-ui=true \\

  --allow-privileged=true \\

  --apiserver-count=3 \\

  --audit-log-maxage=30 \\

  --audit-log-maxbackup=3 \\

  --audit-log-maxsize=100 \\

  --audit-log-path=/var/log/kube-apiserver-audit.log \\

  --event-ttl=1h \\

  --alsologtostderr=true \\

  --logtostderr=false \\

  --log-dir=/var/log/kube \\

  --v=2

Restart=on-failure

RestartSec=5

Type=notify

LimitNOFILE=65536

[Install]

WantedBy=multi-user.target

EOF

systemctl daemon-reload

systemctl start kube-apiserver

systemctl enable kube-apiserver

systemctl status kube-apiserver

#检查测试

ETCDCTL_API=3 etcdctl --endpoints=${ETCD_ENDPOINTS} get /registry/ --prefix --keys-only

kubectl cluster-info

#Kubernetes master is running at https://172.27.129.253:8443

#To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.

kubectl get all --all-namespaces

#NAMESPACE   NAME                 TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)   AGE

#default     service/kubernetes   ClusterIP   10.254.0.1   <none>        443/TCP   35m

kubectl get componentstatuses

#NAME                 STATUS      MESSAGE                                                                                        ERROR

#controller-manager   Unhealthy   Get http://127.0.0.1:10252/healthz: dial tcp 127.0.0.1:10252: getsockopt: connection refused

#scheduler            Unhealthy   Get http://127.0.0.1:10251/healthz: dial tcp 127.0.0.1:10251: getsockopt: connection refused

#etcd-1               Healthy     {"health":"true"}

#etcd-0               Healthy     {"health":"true"}

#etcd-2               Healthy     {"health":"true"}

netstat -lnpt|grep kube

#授予 kubernetes 证书访问 kubelet API 的权限

kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=systemdm:kubelet-api-admin --user kubernetes

#部署kube-controller-manager 

#创建和分发 kube-controller-manager systemd unit 文件

source /opt/k8s/env.sh

cd /etc/systemd/system

cat > kube-controller-manager.service <<EOF

[Unit]

Description=Kubernetes Controller Manager

Documentation=https://github.com/GoogleCloudPlatform/kubernetes

[Service]

ExecStart=/usr/local/bin/kube-controller-manager \\

  --address=127.0.0.1 \\

  --master=http://${MASTER_IP}:8080 \\

  --allocate-node-cidrs=true \\

  --service-cluster-ip-range=${SERVICE_CIDR} \\

  --cluster-cidr=${CLUSTER_CIDR} \\

  --cluster-name=kubernetes \\

  --cluster-signing-cert-file=/etc/kubernetes/cert/ca.pem \\

  --cluster-signing-key-file=/etc/kubernetes/cert/ca-key.pem \\

  --service-account-private-key-file=/etc/kubernetes/cert/ca-key.pem \\

  --root-ca-file=/etc/kubernetes/cert/ca.pem \\

  --feature-gates=RotateKubeletServerCertificate=true \\

  --controllers=*,bootstrapsigner,tokencleaner \\

  --use-service-account-credentials=true \\

  --leader-elect=true \\

  --alsologtostderr=true \\

  --logtostderr=false \\

  --log-dir=/var/log/kube \\

  --v=2

Restart=on-failure

RestartSec=5

[Install]

WantedBy=multi-user.target

EOF

systemctl daemon-reload

systemctl enable kube-controller-manager

systemctl restart kube-controller-manager

#检查

systemctl status kube-controller-manager

netstat -lnpt|grep kube-controll

#curl -s --cacert /etc/kubernetes/cert/ca.pem https://127.0.0.1:10252/metrics |head

#部署kube-scheduler 

#在安全端口(https，10251) 输出 prometheus 格式的 metrics；

#创建 kube-scheduler 证书和私钥

#创建证书签名请求：

cd /home

#创建和分发 kube-scheduler systemd unit 文件

source /opt/k8s/env.sh

cd /etc/systemd/system

cat > kube-scheduler.service <<EOF

[Unit]

Description=Kubernetes Scheduler

Documentation=https://github.com/GoogleCloudPlatform/kubernetes

[Service]

ExecStart=/usr/local/bin/kube-scheduler \\

  --address=127.0.0.1 \\

  --master=http://${MASTER_IP}:8080 \\

  --leader-elect=true \\

  --v=2

Restart=on-failure

RestartSec=5

[Install]

WantedBy=multi-user.target

EOF

systemctl daemon-reload

systemctl enable kube-scheduler

systemctl restart kube-scheduler

#检查

systemctl status kube-scheduler

netstat -lnpt|grep kube-sche

#curl -s http://127.0.0.1:10251/metrics |head

#######################-----部署 worker 节点-----#######################

#三台服务器都执行

#安装Docker(使用yum的方式)

#安装docker

#删除旧版本

yum remove docker \

                  docker-client \

                  docker-client-latest \

                  docker-common \

                  docker-latest \

                  docker-latest-logrotate \

                  docker-logrotate \

                  docker-selinux \

                  docker-engine-selinux \

                  docker-engine

yum install -y yum-utils device-mapper-persistent-data lvm2

yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo

yum-config-manager --enable docker-ce-stable

yum-config-manager --disable docker-ce-test

yum-config-manager --disable docker-ce-edge

yum install docker-ce -y

rm -rf /usr/lib/systemd/system/docker.service

vim /usr/lib/systemd/system/docker.service

#输入以下内容

#cat > docker.service <<"EOF"

[Unit]

Description=Docker Application Container Engine

Documentation=http://docs.docker.io

[Service]

EnvironmentFile=-/run/flannel/docker

ExecStart=/usr/bin/dockerd --log-level=info $DOCKER_NETWORK_OPTIONS

ExecReload=/bin/kill -s HUP $MAINPID

Restart=on-failure

RestartSec=5

LimitNOFILE=infinity

LimitNPROC=infinity

LimitCORE=infinity

Delegate=yes

KillMode=process

[Install]

WantedBy=multi-user.target

#EOF

systemctl daemon-reload

systemctl start docker.service

systemctl enable docker.service

#检查

systemctl status docker.service

ip addr show flannel.1 && ip addr show docker0

#部署 kubelet 组件

#下载和分发 kubelet 二进制文件(3.150单独执行)

cd /home/kubernetes

scp /home/kubernetes/server/bin/{kube-proxy,kubelet,kubeadm} root@172.16.3.150:/usr/local/bin/

scp /home/kubernetes/server/bin/{kube-proxy,kubelet,kubeadm} root@172.16.3.151:/usr/local/bin/

scp /home/kubernetes/server/bin/{kube-proxy,kubelet,kubeadm} root@172.16.3.152:/usr/local/bin/

#创建 kubelet bootstrap kubeconfig 文件(3.150单独执行)

source /opt/k8s/env.sh

for node_name in ${NODE_NAMES[@]}

  do

    echo ">>> ${node_name}"

    # 创建 token

    export BOOTSTRAP_TOKEN=$(kubeadm token create \

      --description kubelet-bootstrap-token \

      --groups system:bootstrappers:${node_name} \

      --kubeconfig ~/.kube/config)

    # 设置集群参数

    kubectl config set-cluster kubernetes \

      --certificate-authority=/etc/kubernetes/cert/ca.pem \

      --embed-certs=true \

      --server=${KUBE_APISERVER} \

      --kubeconfig=kubelet-bootstrap-${node_name}.kubeconfig

    # 设置客户端认证参数

    kubectl config set-credentials kubelet-bootstrap \

      --token=${BOOTSTRAP_TOKEN} \

      --kubeconfig=kubelet-bootstrap-${node_name}.kubeconfig

    # 设置上下文参数

    kubectl config set-context default \

      --cluster=kubernetes \

      --user=kubelet-bootstrap \

      --kubeconfig=kubelet-bootstrap-${node_name}.kubeconfig

    # 设置默认上下文

    kubectl config use-context default --kubeconfig=kubelet-bootstrap-${node_name}.kubeconfig

done

kubeadm token list --kubeconfig ~/.kube/config

#创建一个 clusterrolebinding，将 group system:bootstrappers 和 clusterrole system:node-bootstrapper 绑定：(3.150执行)

kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --group=system:bootstrappers

#分发 bootstrap kubeconfig 文件到所有 worker 节点(3.150单独执行)

scp kubelet-bootstrap-dev-test-3-150.kubeconfig root@172.16.3.150:/etc/kubernetes/kubelet-bootstrap.kubeconfig

scp kubelet-bootstrap-dev-test-3-151.kubeconfig root@172.16.3.151:/etc/kubernetes/kubelet-bootstrap.kubeconfig

scp kubelet-bootstrap-dev-test-3-152.kubeconfig root@172.16.3.152:/etc/kubernetes/kubelet-bootstrap.kubeconfig

#创建和分发 kubelet 参数配置文件(三台服务器都执行)

source /opt/k8s/env.sh

cd /etc/kubernetes

cat > kubelet.config.json <<EOF

{

  "kind": "KubeletConfiguration",

  "apiVersion": "kubelet.config.k8s.io/v1beta1",

  "authentication": {

    "x509": {

      "clientCAFile": "/etc/kubernetes/cert/ca.pem"

    },

    "webhook": {

      "enabled": true,

      "cacheTTL": "2m0s"

    },

    "anonymous": {

      "enabled": false

    }

  },

  "authorization": {

    "mode": "Webhook",

    "webhook": {

      "cacheAuthorizedTTL": "5m0s",

      "cacheUnauthorizedTTL": "30s"

    }

  },

  "address": "${NODE_IP}",

  "port": 10250,

  "readOnlyPort": 0,

  "cgroupDriver": "cgroupfs",

  "hairpinMode": "promiscuous-bridge",

  "serializeImagePulls": false,

  "featureGates": {

    "RotateKubeletClientCertificate": true,

    "RotateKubeletServerCertificate": true

  },

  "clusterDomain": "${CLUSTER_DNS_DOMAIN}",

  "clusterDNS": ["${CLUSTER_DNS_SVC_IP}"]

}

EOF

#创建 kubelet systemd unit 文件(三台服务器都执行)

source /opt/k8s/env.sh

mkdir /var/lib/kubelet

cd /etc/systemd/system

cat > kubelet.service <<EOF

[Unit]

Description=Kubernetes Kubelet

Documentation=https://github.com/GoogleCloudPlatform/kubernetes

After=docker.service

Requires=docker.service

[Service]

WorkingDirectory=/var/lib/kubelet

ExecStart=/usr/local/bin/kubelet \\

  --bootstrap-kubeconfig=/etc/kubernetes/kubelet-bootstrap.kubeconfig \\

  --cert-dir=/etc/kubernetes/cert \\

  --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \\

  --config=/etc/kubernetes/kubelet.config.json \\

  --hostname-override=${NODE_NAME} \\

  --pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 \\

  --allow-privileged=true \\

  --alsologtostderr=true \\

  --logtostderr=false \\

  --log-dir=/var/log/kube \\

  --v=2

Restart=on-failure

RestartSec=5

[Install]

WantedBy=multi-user.target

EOF

其他节点启动失败的话，检查一下防火墙

iptables -F &&  iptables -X &&  iptables -F -t nat &&  iptables -X -t nat

iptables -P FORWARD ACCEPT

systemctl daemon-reload

systemctl start kubelet.service

systemctl enable kubelet.service

#检查

systemctl status kubelet.service

#approve kubelet CSR 请求(3.150执行)

kubectl get csr

#NAME                                                   AGE       REQUESTOR                 CONDITION

#node-csr-LY1jhbfkcVBthGoveaEhUycp8yg_7D4atQkmQSz3N3Y   13m       system:bootstrap:faum34   Pending

#node-csr-WY5RcykyS66oKc_wHydYiK6QkAFDluCIP856QB_0QRM   6m        system:bootstrap:aedkze   Pending

#node-csr-bwIGn4TJECVhWUIU5j0ckIAtzR5Qkgt8ZQzuHpg-NcA   6m        system:bootstrap:esigku   Pending

#手动 approve CSR 请求(3.150执行)

for i in `kubectl get csr | grep -v NAME | awk '{print $1}'`; do kubectl certificate approve ${i}; done

#查看节点

kubectl get nodes

#NAME             STATUS    ROLES     AGE       VERSION

#dev-test-3-150   Ready     <none>    27s       v1.10.4

#dev-test-3-151   Ready     <none>    27s       v1.10.4

#dev-test-3-152   Ready     <none>    25s       v1.10.4

#创建自动approve csr(3.150执行)

cd /home

source /opt/k8s/env.sh

cat > csr-crb.yaml <<EOF

 # Approve all CSRs for the group "system:bootstrappers"

 kind: ClusterRoleBinding

 apiVersion: rbac.authorization.k8s.io/v1

 metadata:

   name: auto-approve-csrs-for-group

 subjects:

 - kind: Group

   name: system:bootstrappers

   apiGroup: rbac.authorization.k8s.io

 roleRef:

   kind: ClusterRole

   name: system:certificates.k8s.io:certificatesigningrequests:nodeclient

   apiGroup: rbac.authorization.k8s.io

---

 # To let a node of the group "system:nodes" renew its own credentials

 kind: ClusterRoleBinding

 apiVersion: rbac.authorization.k8s.io/v1

 metadata:

   name: node-client-cert-renewal

 subjects:

 - kind: Group

   name: system:nodes

   apiGroup: rbac.authorization.k8s.io

 roleRef:

   kind: ClusterRole

   name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient

   apiGroup: rbac.authorization.k8s.io

---

# A ClusterRole which instructs the CSR approver to approve a node requesting a

# serving cert matching its client cert.

kind: ClusterRole

apiVersion: rbac.authorization.k8s.io/v1

metadata:

  name: approve-node-server-renewal-csr

rules:

- apiGroups: ["certificates.k8s.io"]

  resources: ["certificatesigningrequests/selfnodeserver"]

  verbs: ["create"]

---

 # To let a node of the group "system:nodes" renew its own server credentials

 kind: ClusterRoleBinding

 apiVersion: rbac.authorization.k8s.io/v1

 metadata:

   name: node-server-cert-renewal

 subjects:

 - kind: Group

   name: system:nodes

   apiGroup: rbac.authorization.k8s.io

 roleRef:

   kind: ClusterRole

   name: approve-node-server-renewal-csr

   apiGroup: rbac.authorization.k8s.io

EOF

#auto-approve-csrs-for-group：自动 approve node 的第一次 CSR； 注意第一次 CSR 时，请求的 Group 为 system:bootstrappers；

#node-client-cert-renewal：自动 approve node 后续过期的 client 证书，自动生成的证书 Group 为 system:nodes;

#node-server-cert-renewal：自动 approve node 后续过期的 server 证书，自动生成的证书 Group 为 system:nodes;

kubectl apply -f csr-crb.yaml

#创建kube-proxy(3.150执行)

#创建 kube-proxy 证书

#创建证书签名请求：

cd /home

cat > kube-proxy-csr.json <<EOF

{

  "CN": "system:kube-proxy",

  "key": {

    "algo": "rsa",

    "size": 2048

  },

  "names": [

    {

      "C": "CN",

      "ST": "BeiJing",

      "L": "BeiJing",

      "O": "k8s",

      "OU": "QRXD"

    }

  ]

}

EOF

cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \

  -ca-key=/etc/kubernetes/cert/ca-key.pem \

  -config=/etc/kubernetes/cert/ca-config.json \

  -profile=kubernetes  kube-proxy-csr.json | cfssljson -bare kube-proxy

source /opt/k8s/env.sh

kubectl config set-cluster kubernetes \

  --certificate-authority=/etc/kubernetes/cert/ca.pem \

  --embed-certs=true \

  --server=${KUBE_APISERVER} \

  --kubeconfig=kube-proxy.kubeconfig

kubectl config set-credentials kube-proxy \

  --client-certificate=kube-proxy.pem \

  --client-key=kube-proxy-key.pem \

  --embed-certs=true \

  --kubeconfig=kube-proxy.kubeconfig

kubectl config set-context default \

  --cluster=kubernetes \

  --user=kube-proxy \

  --kubeconfig=kube-proxy.kubeconfig

kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

#分发配置文件

scp kube-proxy.kubeconfig root@172.16.3.150:/etc/kubernetes/

scp kube-proxy.kubeconfig root@172.16.3.151:/etc/kubernetes/

scp kube-proxy.kubeconfig root@172.16.3.152:/etc/kubernetes/

#创建 kube-proxy 配置文件(三台都执行)

cd /etc/kubernetes

source /opt/k8s/env.sh

cat >kube-proxy.config.yaml <<EOF

apiVersion: kubeproxy.config.k8s.io/v1alpha1

bindAddress: ${NODE_IP}

clientConnection:

  kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig

clusterCIDR: ${CLUSTER_CIDR}

healthzBindAddress: ${NODE_IP}:10256

hostnameOverride: ${NODE_NAME}

kind: KubeProxyConfiguration

metricsBindAddress: ${NODE_IP}:10249

mode: "ipvs"

EOF

#创建 kube-proxy systemd unit 文件(三台都执行)

mkdir /var/lib/kube-proxy

source /opt/k8s/env.sh

cd /etc/systemd/system 

cat > kube-proxy.service <<EOF

[Unit]

Description=Kubernetes Kube-Proxy Server

Documentation=https://github.com/GoogleCloudPlatform/kubernetes

After=network.target

[Service]

WorkingDirectory=/var/lib/kube-proxy

ExecStart=/usr/local/bin/kube-proxy \\

  --config=/etc/kubernetes/kube-proxy.config.yaml \\

  --alsologtostderr=true \\

  --hostname-override=${NODE_NAME} \\

  --logtostderr=false \\

  --log-dir=/var/log/kube \\

  --v=2

Restart=on-failure

RestartSec=5

LimitNOFILE=65536

[Install]

WantedBy=multi-user.target

EOF

systemctl daemon-reload

systemctl enable kube-proxy

systemctl restart kube-proxy

#检查

systemctl status kube-proxy

netstat -lnpt|grep kube-proxy

#查看路由规则

source /opt/k8s/env.sh

for node_ip in ${NODE_IPS[@]}

  do

    echo ">>> ${node_ip}"

    ssh root@${node_ip} "/usr/sbin/ipvsadm -ln"

done

#######################-----验证集群功能-----#######################

kubectl get nodes

#测试文件

cd /home

mkdir yaml

cd yaml

cat > nginx-ds.yml <<EOF

apiVersion: v1

kind: Service

metadata:

  name: nginx-ds

  labels:

    app: nginx-ds

spec:

  type: NodePort

  selector:

    app: nginx-ds

  ports:

  - name: http

    port: 80

    targetPort: 80

---

apiVersion: extensions/v1beta1

kind: DaemonSet

metadata:

  name: nginx-ds

  labels:

    addonmanager.kubernetes.io/mode: Reconcile

spec:

  template:

    metadata:

      labels:

        app: nginx-ds

    spec:

      containers:

      - name: my-nginx

        image: nginx:1.7.9

        ports:

        - containerPort: 80

EOF

kubectl apply -f nginx-ds.yml -n default

kubectl get pod

#######################-----部署集群插件-----#######################

#-------------------------------

#部署 DNS 插件(coredns)

mkdir /home/yaml/coredns

cp /home/kubernetes/cluster/addons/dns/coredns.yaml.base /home/yaml/coredns/coredns.yaml

#修改信息如下:

#61行

__PILLAR__DNS__DOMAIN__  ==> cluster.local.

#153行

__PILLAR__DNS__SERVER__  ==> 10.253.0.2

#创建coredns

kubectl apply -f /home/kube-system/coredns.yaml -n kube-system

#检查

kubectl get all -n kube-system

#-------------------------------

#部署 DNS 插件(sky-dns)

mkdir /home/kube-system/kube-dns

cp  /home/kubernetes/cluster/addons/dns/

#-------------------------------

#部署 dashboard 插件

mkdir /home/yaml/dashboard

#yaml文件地址:

#官方v1.10.0版本

#wget https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/recommended/kubernetes-dashboard.yaml

#修改版v1.8.3版本

#wget https://github.com/gh-Devin/kubernetes-dashboard/blob/master/kubernetes-dashboard.yaml

#镜像名称也需要修改下根据自己的镜像修改需要提前下载下来

#kubectl apply -f kubernetes-dashboard.yaml

cd /home/kubernetes/cluster/addons/dashboard

cp dashboard-* /home/kubernetes/cluster/addons/dashboard/

vim dashboard-service.yaml

#末尾新增,并和ports节对齐

type: NodePort

#创建dashboard

kubectl applf -f .

#查看 kubectl get pod -n kube-system

kubectl get pod -n kube-system

kubectl get svc -n kube-system

kubectl cluster-info

#Kubernetes master is running at https://172.16.3.150:6443

#CoreDNS is running at https://172.16.3.150:6443/api/v1/namespaces/kube-system/services/coredns:dns/proxy

#kubernetes-dashboard is running at https://172.16.3.150:6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy

#注意，只有cluster-info 中可以看到kubernetes-dashboard is running .... ，才可以通过kube-apiserver去访问dashboard

#创建登录 Dashboard 的 token 和 kubeconfig 配置文件

cd /home

kubectl create sa dashboard-admin -n kube-system

kubectl create clusterrolebinding dashboard-admin --clusterrole=cluster-admin --serviceaccount=kube-system:dashboard-admin

ADMIN_SECRET=$(kubectl get secrets -n kube-system | grep dashboard-admin | awk '{print $1}')

DASHBOARD_LOGIN_TOKEN=$(kubectl describe secret -n kube-system ${ADMIN_SECRET} | grep -E '^token' | awk '{print $2}')

echo ${DASHBOARD_LOGIN_TOKEN}

#可以使用输出的 token 登录 Dashboard

#创建使用 token 的 KubeConfig 文件‘

source /opt/k8s/env.sh

cd /home

kubectl config set-cluster kubernetes \

  --certificate-authority=/etc/kubernetes/cert/ca.pem \

  --embed-certs=true \

  --server=${KUBE_APISERVER} \

  --kubeconfig=dashboard.kubeconfig

# 设置客户端认证参数，使用上面创建的 Token

kubectl config set-credentials dashboard_user \

  --token=${DASHBOARD_LOGIN_TOKEN} \

  --kubeconfig=dashboard.kubeconfig

# 设置上下文参数

kubectl config set-context default \

  --cluster=kubernetes \

  --user=dashboard_user \

  --kubeconfig=dashboard.kubeconfig

# 设置默认上下文

kubectl config use-context default --kubeconfig=dashboard.kubeconfig

#导出dashboard.kubeconfig在登录页面进行上传

#配置允许浏览器访问(3.150)

cd /home

openssl pkcs12 -export -out admin.pfx -inkey admin-key.pem -in admin.pem -certfile ca.pem

#需要输入证书密码

#导出admin.pfx

#将admin.pfx导入到本地的证书管理中(windows通过 mmc管理单元配置)

#浏览器打开以下地址

https://172.16.3.150:6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy

#使用nodeport访问

kubectl get svc -n kube-system

#查看到对应的nodeport

#通过浏览器访问(用Firefox访问，用chrom会有些问题)

https://172.16.3.150:13054

#登录时需要使用导出的dashboard.kubeconfig 进行验证

#-------------------------------

#部署 heapster 插件

#提前把需要的镜像下载下来

cd /home/

wget https://github.com/kubernetes/heapster/archive/v1.5.3.tar.gz

tar -xzvf v1.5.3.tar.gz

mkdir /home/yaml/heapster

cp /home/heapster-1.5.3/deploy/kube-config/influxdb/* /home/yaml/heapster/

#cp /home/heapster-1.5.3/deploy/kube-config/rbac/* /home/yaml/heapster/

cd /home/yaml/heapster

vim grafana.yaml

#67行注释去掉，位置和ports对齐

type: NodePort

vim heapster.yaml

#27行 修改成如下

- --source=kubernetes:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250

#influxdb 不需要修改

cat > heapster-rbac.yaml <<EOF

kind: ClusterRoleBinding

apiVersion: rbac.authorization.k8s.io/v1beta1

metadata:

  name: heapster-kubelet-api

roleRef:

  apiGroup: rbac.authorization.k8s.io

  kind: ClusterRole

  #name: cluster-admin

  name: system:kubelet-api-admin

subjects:

- kind: ServiceAccount

  name: heapster

  namespace: kube-system

EOF

#将 serviceAccount kube-system:heapster 与 ClusterRole system:kubelet-api-admin 绑定，授予它调用 kubelet API 的权限；

kubectl apply -f .

#可以通过node port进行访问

kubectl get svc -n kube-system|grep -E 'monitoring|heapster'

#错误信息:1 namespace based enricher.go:75] Namespace doesn't exist: default

#权限不够，rbac授权 可以使用cluster-admin 角色解决该问题

#-------------------------------

#安装prometheus

#Prometheus 可以通过服务发现掌握集群内部已经暴露的监控点，然后主动拉取所有监控数据。

#通过这样的架构设计，我们仅仅只需要向Kubernetes集群中部署一份Prometheus实例，

#它就可以通过向apiserver查询集群状态，然后向所有已经支持Prometheus metrics的kubelet 获取所有Pod的运行数据。如果我们想采集底层服务器运行状态，

#通过DaemonSet在所有服务器上运行 配套的node-exporter之后，Prometheus就可以自动采集到新的这部分数据。

#详细配置解释在配置文件中说明

##node-exporter.yaml

cat node-exporter.yaml <<EOF

---

apiVersion: extensions/v1beta1

kind: DaemonSet

metadata:

  name: node-exporter

  #指定运行在kube-system中,也可以根据自己的配置指定其他的namespace

  namespace: kube-system

  labels:

    k8s-app: node-exporter

spec:

  template:

    metadata:

      labels:

        k8s-app: node-exporter

    spec:

      containers:

      #镜像不需要×××就可以下载

      - image: prom/node-exporter

        name: node-exporter

        ports:

        - containerPort: 9100

          protocol: TCP

          name: http

---

apiVersion: v1

kind: Service

metadata:

  labels:

    k8s-app: node-exporter

  name: node-exporter

  namespace: kube-system

spec:

  ports:

  - name: http

    port: 9100

    nodePort: 21672

    protocol: TCP

  type: NodePort

  selector:

    k8s-app: node-exporter

EOF

##prometheus-configmap.yaml

cat prometheus-configmap.yaml <<EOF

apiVersion: v1

kind: ConfigMap

metadata:

  name: prometheus-config

  #此处的namespace需要和node-exporter一致

  namespace: kube-system

data:

  prometheus.yml: |

    global:

      scrape_interval: 30s

      scrape_timeout: 30s

    scrape_configs:

    - job_name: 'prometheus'

      static_configs:

        - targets: ['localhost:9090']

    - job_name: 'kubernetes-apiservers'

      kubernetes_sd_configs:

      - role: endpoints

      scheme: https

      tls_config:

        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

      relabel_configs:

      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]

        action: keep

        regex: default;kubernetes;https

    - job_name: 'kubernetes-nodes'

      scheme: https

      tls_config:

        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

      kubernetes_sd_configs:

      - role: node

      relabel_configs:

      - action: labelmap

        regex: __meta_kubernetes_node_label_(.+)

      - target_label: __address__

        replacement: kubernetes.default.svc:443

      - source_labels: [__meta_kubernetes_node_name]

        regex: (.+)

        target_label: __metrics_path__

        replacement: /api/v1/nodes/${1}/proxy/metrics

    - job_name: 'kubernetes-cadvisor'

      scheme: https

      tls_config:

        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

      kubernetes_sd_configs:

      - role: node

      relabel_configs:

      - action: labelmap

        regex: __meta_kubernetes_node_label_(.+)

      - target_label: __address__

        replacement: kubernetes.default.svc:443

      - source_labels: [__meta_kubernetes_node_name]

        regex: (.+)

        target_label: __metrics_path__

        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor

    - job_name: 'kubernetes-node-exporter'

      scheme: http

      tls_config:

        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

      kubernetes_sd_configs:

      - role: node

      relabel_configs:

      - action: labelmap

        regex: __meta_kubernetes_node_label_(.+)

      - source_labels: [__meta_kubernetes_role]

        action: replace

        target_label: kubernetes_role

      - source_labels: [__address__]

        regex: '(.*):10250'

        #下面这个端口需要注意，这个地址是实际的node-exporter的nodeport端口

        replacement: '${1}:21672'

        target_label: __address__

EOF

##prometheus.yaml

cat >prometheus.yaml <<EOF

apiVersion: extensions/v1beta1

kind: Deployment

metadata:

  labels:

    k8s-app: prometheus

  name: prometheus

  namespace: kube-system

spec:

  replicas: 1

  template:

    metadata:

      labels:

        k8s-app: prometheus

    spec:

      serviceAccountName: prometheus

      containers:

      - image: prom/prometheus:v2.0.0

        name: prometheus

        command:

        - "/bin/prometheus"

        args:

        - "--config.file=/etc/prometheus/prometheus.yml"

        - "--storage.tsdb.path=/prometheus"

        - "--storage.tsdb.retention=15d"

        ports:

        - containerPort: 9090

          protocol: TCP

          name: http

        volumeMounts:

        - mountPath: "/prometheus"

          name: data

          subPath: prometheus/data

        - mountPath: "/etc/prometheus"

          name: config-volume

        resources:

          requests:

            cpu: 100m

            memory: 100Mi

          limits:

            cpu: 200m

            memory: 1Gi

      volumes:

      - name: data

        emptyDir: {}

      - configMap:

          name: prometheus-config

        name: config-volume

EOF

##prometheus-rabc.yaml

cat >prometheus-rabc.yaml<<EOF

apiVersion: v1

kind: ServiceAccount

metadata:

  name: prometheus

  namespace: kube-system

---

apiVersion: rbac.authorization.k8s.io/v1beta1

kind: ClusterRole

metadata:

  name: prometheus

  namespace: kube-system

rules:

- apiGroups: [""]

  resources:

  - nodes

  - nodes/proxy

  - services

  - endpoints

  - pods

  verbs: ["get", "list", "watch"]

- nonResourceURLs: ["/metrics"]

  verbs: ["get"]

---

apiVersion: rbac.authorization.k8s.io/v1beta1

kind: ClusterRoleBinding

metadata:

  name: prometheus

  namespace: kube-system

roleRef:

  apiGroup: rbac.authorization.k8s.io

  kind: ClusterRole

  name: prometheus

subjects:

- kind: ServiceAccount

  name: prometheus

  namespace: kube-system

EOF

##prometheus-service.yaml

cat >prometheus-service.yaml <<EOF

apiVersion: v1

kind: Service

metadata:

  namespace: kube-system

  name: prometheus-svc

  labels:

    k8s-app: prometheus

spec:

  selector:

    k8s-app: prometheus

  ports:

  - port: 9090

    targetPort: 9090

  type: NodePort

EOF

kubectl apply -f .

#-------------------------------

#######################-----Kubernetes镜像相关-----#######################

#首先下载镜像

docker pull majun9129/kubernetes-images:pause-3.0

docker pull majun9129/kubernetes-images:kube-dashboard-1.10.0 

docker pull majun9129/kubernetes-images:dashboard-1.8.3

docker pull majun9129/kubernetes-images:kube-heapster-influxdb-1.3.3

docker pull majun9129/kubernetes-images:kube-heapster-grafana-4.4.3

docker pull majun9129/kubernetes-images:kube-heapster-1.5.3

#把镜像重命名为官网镜像名称

docker tag majun9129/kubernetes-images:pause-3.0 gcr.io/google_containers/pause-amd64:3.0

docker tag  majun9129/kubernetes-images:kube-dashboard-1.10.0  k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.0

docker tag majun9129/kubernetes-images:dashboard-1.8.3 k8s.gcr.io/kubernetes-dashboard-amd64:v1.8.3

docker tag majun9129/kubernetes-images:kube-heapster-influxdb-1.3.3 gcr.io/google_containers/heapster-influxdb-amd64:v1.3.3

docker tag majun9129/kubernetes-images:kube-heapster-grafana-4.4.3 gcr.io/google_containers/heapster-grafana-amd64:v4.4.3

docker tag majun9129/kubernetes-images:kube-heapster-1.5.3 gcr.io/google_containers/heapster-amd64:v1.5.3

#######################-----Kubernetes 插件yaml文件-----#######################

#####core-dns#####

cat >coredns.yaml <<EOF

# __MACHINE_GENERATED_WARNING__

apiVersion: v1

kind: ServiceAccount

metadata:

  name: coredns

  namespace: kube-system

  labels:

      kubernetes.io/cluster-service: "true"

      addonmanager.kubernetes.io/mode: Reconcile

---

apiVersion: rbac.authorization.k8s.io/v1

kind: ClusterRole

metadata:

  labels:

    kubernetes.io/bootstrapping: rbac-defaults

    addonmanager.kubernetes.io/mode: Reconcile

  name: system:coredns

rules:

- apiGroups:

  - ""

  resources:

  - endpoints

  - services

  - pods

  - namespaces

  verbs:

  - list

  - watch

---

apiVersion: rbac.authorization.k8s.io/v1

kind: ClusterRoleBinding

metadata:

  annotations:

    rbac.authorization.kubernetes.io/autoupdate: "true"

  labels:

    kubernetes.io/bootstrapping: rbac-defaults

    addonmanager.kubernetes.io/mode: EnsureExists

  name: system:coredns

roleRef:

  apiGroup: rbac.authorization.k8s.io

  kind: ClusterRole

  name: system:coredns

subjects:

- kind: ServiceAccount

  name: coredns

  namespace: kube-system

---

apiVersion: v1

kind: ConfigMap

metadata:

  name: coredns

  namespace: kube-system

  labels:

      addonmanager.kubernetes.io/mode: EnsureExists

data:

  Corefile: |

    .:53 {

        errors

        health

        kubernetes cluster.local. in-addr.arpa ip6.arpa {

            pods insecure

            upstream

            fallthrough in-addr.arpa ip6.arpa

        }

        prometheus :9153

        proxy . /etc/resolv.conf

        cache 30

    }

---

apiVersion: extensions/v1beta1

kind: Deployment

metadata:

  name: coredns

  namespace: kube-system

  labels:

    k8s-app: coredns

    kubernetes.io/cluster-service: "true"

    addonmanager.kubernetes.io/mode: Reconcile

    kubernetes.io/name: "CoreDNS"

spec:

  replicas: 3

  strategy:

    type: RollingUpdate

    rollingUpdate:

      maxUnavailable: 1

  selector:

    matchLabels:

      k8s-app: coredns

  template:

    metadata:

      labels:

        k8s-app: coredns

    spec:

      serviceAccountName: coredns

      tolerations:

        - key: node-role.kubernetes.io/master

          effect: NoSchedule

        - key: "CriticalAddonsOnly"

          operator: "Exists"

      containers:

      - name: coredns

        image: coredns/coredns:1.0.6

        imagePullPolicy: IfNotPresent

        resources:

          limits:

            memory: 170Mi

          requests:

            cpu: 100m

            memory: 70Mi

        args: [ "-conf", "/etc/coredns/Corefile" ]

        volumeMounts:

        - name: config-volume

          mountPath: /etc/coredns

        ports:

        - containerPort: 53

          name: dns

          protocol: UDP

        - containerPort: 53

          name: dns-tcp

          protocol: TCP

        livenessProbe:

          httpGet:

            path: /health

            port: 8080

            scheme: HTTP

          initialDelaySeconds: 60

          timeoutSeconds: 5

          successThreshold: 1

          failureThreshold: 5

      dnsPolicy: Default

      volumes:

        - name: config-volume

          configMap:

            name: coredns

            items:

            - key: Corefile

              path: Corefile

---

apiVersion: v1

kind: Service

metadata:

  name: coredns

  namespace: kube-system

  labels:

    k8s-app: coredns

    kubernetes.io/cluster-service: "true"

    addonmanager.kubernetes.io/mode: Reconcile

    kubernetes.io/name: "CoreDNS"

spec:

  selector:

    k8s-app: coredns

  clusterIP: 10.253.0.2

  ports:

  - name: dns

    port: 53

    protocol: UDP

  - name: dns-tcp

    port: 53

    protocol: TCP

EOF

###dashboard###

cat >dashboard-configmap.yaml <<EOF

apiVersion: v1

kind: ConfigMap

metadata:

  labels:

    k8s-app: kubernetes-dashboard

    # Allows editing resource and makes sure it is created first.

    addonmanager.kubernetes.io/mode: EnsureExists

  name: kubernetes-dashboard-settings

  namespace: kube-system

EOF

cat >dashboard-controller.yaml <<EOF

apiVersion: v1

kind: ServiceAccount

metadata:

  labels:

    k8s-app: kubernetes-dashboard

    addonmanager.kubernetes.io/mode: Reconcile

  name: kubernetes-dashboard

  namespace: kube-system

---

apiVersion: apps/v1

kind: Deployment

metadata:

  name: kubernetes-dashboard

  namespace: kube-system

  labels:

    k8s-app: kubernetes-dashboard

    kubernetes.io/cluster-service: "true"

    addonmanager.kubernetes.io/mode: Reconcile

spec:

  selector:

    matchLabels:

      k8s-app: kubernetes-dashboard

  template:

    metadata:

      labels:

        k8s-app: kubernetes-dashboard

      annotations:

        scheduler.alpha.kubernetes.io/critical-pod: ''

    spec:

      priorityClassName: system-cluster-critical

      containers:

      - name: kubernetes-dashboard

        image: k8s.gcr.io/kubernetes-dashboard-amd64:v1.8.3

        resources:

          limits:

            cpu: 100m

            memory: 300Mi

          requests:

            cpu: 50m

            memory: 100Mi

        ports:

        - containerPort: 8443

          protocol: TCP

        args:

          # PLATFORM-SPECIFIC ARGS HERE

          - --auto-generate-certificates

        volumeMounts:

        - name: kubernetes-dashboard-certs

          mountPath: /certs

        - name: tmp-volume

          mountPath: /tmp

        livenessProbe:

          httpGet:

            scheme: HTTPS

            path: /

            port: 8443

          initialDelaySeconds: 30

          timeoutSeconds: 30

      volumes:

      - name: kubernetes-dashboard-certs

        secret:

          secretName: kubernetes-dashboard-certs

      - name: tmp-volume

        emptyDir: {}

      serviceAccountName: kubernetes-dashboard

      tolerations:

      - key: "CriticalAddonsOnly"

        operator: "Exists"

EOF

cat >dashboard-rbac.yaml <<EOF

kind: Role

apiVersion: rbac.authorization.k8s.io/v1

metadata:

  labels:

    k8s-app: kubernetes-dashboard

    addonmanager.kubernetes.io/mode: Reconcile

  name: kubernetes-dashboard-minimal

  namespace: kube-system

rules:

  # Allow Dashboard to get, update and delete Dashboard exclusive secrets.

- apiGroups: [""]

  resources: ["secrets"]

  resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs"]

  verbs: ["get", "update", "delete"]

  # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.

- apiGroups: [""]

  resources: ["configmaps"]

  resourceNames: ["kubernetes-dashboard-settings"]

  verbs: ["get", "update"]

  # Allow Dashboard to get metrics from heapster.

- apiGroups: [""]

  resources: ["services"]

  resourceNames: ["heapster"]

  verbs: ["proxy"]

- apiGroups: [""]

  resources: ["services/proxy"]

  resourceNames: ["heapster", "http:heapster:", "https:heapster:"]

  verbs: ["get"]

---

apiVersion: rbac.authorization.k8s.io/v1

kind: RoleBinding

metadata:

  name: kubernetes-dashboard-minimal

  namespace: kube-system

  labels:

    k8s-app: kubernetes-dashboard

    addonmanager.kubernetes.io/mode: Reconcile

roleRef:

  apiGroup: rbac.authorization.k8s.io

  kind: Role

  name: kubernetes-dashboard-minimal

subjects:

- kind: ServiceAccount

  name: kubernetes-dashboard

  namespace: kube-system

EOF

cat >dashboard-secret.yaml <<EOF

apiVersion: v1

kind: Secret

metadata:

  labels:

    k8s-app: kubernetes-dashboard

    # Allows editing resource and makes sure it is created first.

    addonmanager.kubernetes.io/mode: EnsureExists

  name: kubernetes-dashboard-certs

  namespace: kube-system

type: Opaque

---

apiVersion: v1

kind: Secret

metadata:

  labels:

    k8s-app: kubernetes-dashboard

    # Allows editing resource and makes sure it is created first.

    addonmanager.kubernetes.io/mode: EnsureExists

  name: kubernetes-dashboard-key-holder

  namespace: kube-system

type: Opaque

EOF

cat >dashboard-service.yaml <<EOF

apiVersion: v1

kind: Service

metadata:

  name: kubernetes-dashboard

  namespace: kube-system

  labels:

    k8s-app: kubernetes-dashboard

    kubernetes.io/cluster-service: "true"

    addonmanager.kubernetes.io/mode: Reconcile

spec:

  selector:

    k8s-app: kubernetes-dashboard

  ports:

  - port: 443

    targetPort: 8443

  type: NodePort

EOF

#####heapster#####

cat >grafana.yaml <<EOF

apiVersion: extensions/v1beta1

kind: Deployment

metadata:

  name: monitoring-grafana

  namespace: kube-system

spec:

  replicas: 1

  template:

    metadata:

      labels:

        task: monitoring

        k8s-app: grafana

    spec:

      containers:

      - name: grafana

        image: gcr.io/google_containers/heapster-grafana-amd64:v4.4.3

        ports:

        - containerPort: 3000

          protocol: TCP

        volumeMounts:

        - mountPath: /etc/ssl/certs

          name: ca-certificates

          readOnly: true

        - mountPath: /var

          name: grafana-storage

        env:

        - name: INFLUXDB_HOST

          value: monitoring-influxdb

        - name: GF_SERVER_HTTP_PORT

          value: "3000"

          # The following env variables are required to make Grafana accessible via

          # the kubernetes api-server proxy. On production clusters, we recommend

          # removing these env variables, setup auth for grafana, and expose the grafana

          # service using a LoadBalancer or a public IP.

        - name: GF_AUTH_BASIC_ENABLED

          value: "false"

        - name: GF_AUTH_ANONYMOUS_ENABLED

          value: "true"

        - name: GF_AUTH_ANONYMOUS_ORG_ROLE

          value: Admin

        - name: GF_SERVER_ROOT_URL

          # If you're only using the API Server proxy, set this value instead:

          # value: /api/v1/namespaces/kube-system/services/monitoring-grafana/proxy

          value: /

      volumes:

      - name: ca-certificates

        hostPath:

          path: /etc/ssl/certs

      - name: grafana-storage

        emptyDir: {}

---

apiVersion: v1

kind: Service

metadata:

  labels:

    # For use as a Cluster add-on (https://github.com/kubernetes/kubernetes/tree/master/cluster/addons)

    # If you are NOT using this as an addon, you should comment out this line.

    kubernetes.io/cluster-service: 'true'

    kubernetes.io/name: monitoring-grafana

  name: monitoring-grafana

  namespace: kube-system

spec:

  # In a production setup, we recommend accessing Grafana through an external Loadbalancer

  # or through a public IP.

  # type: LoadBalancer

  # You could also use NodePort to expose the service at a randomly-generated port

  type: NodePort

  ports:

  - port: 80

    targetPort: 3000

  selector:

    k8s-app: grafana

EOF

cat >heapster.yaml <<EOF

apiVersion: v1

kind: ServiceAccount

metadata:

  name: heapster

  namespace: kube-system

---

apiVersion: extensions/v1beta1

kind: Deployment

metadata:

  name: heapster

  namespace: kube-system

spec:

  replicas: 1

  template:

    metadata:

      labels:

        task: monitoring

        k8s-app: heapster

    spec:

      serviceAccountName: heapster

      containers:

      - name: heapster

        image: gcr.io/google_containers/heapster-amd64:v1.5.3

        imagePullPolicy: IfNotPresent

        command:

        - /heapster

        - --source=kubernetes:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250

        #- --source=kubernetes:https://kubernetes.default

        - --sink=influxdb:http://monitoring-influxdb.kube-system.svc:8086

---

apiVersion: v1

kind: Service

metadata:

  labels:

    task: monitoring

    # For use as a Cluster add-on (https://github.com/kubernetes/kubernetes/tree/master/cluster/addons)

    # If you are NOT using this as an addon, you should comment out this line.

    kubernetes.io/cluster-service: 'true'

    kubernetes.io/name: Heapster

  name: heapster

  namespace: kube-system

spec:

  ports:

  - port: 80

    targetPort: 8082

  selector:

    k8s-app: heapster

---

kind: ClusterRoleBinding

apiVersion: rbac.authorization.k8s.io/v1beta1

metadata:

  name: heapster-kubelet-api

roleRef:

  apiGroup: rbac.authorization.k8s.io

  kind: ClusterRole

  #name: system:kubelet-api-admin

  name: cluster-admin

subjects:

- kind: ServiceAccount

  name: heapster

  namespace: kube-system

EOF

cat >influxdb.yaml <<EOF

apiVersion: extensions/v1beta1

kind: Deployment

metadata:

  name: monitoring-influxdb

  namespace: kube-system

spec:

  replicas: 1

  template:

    metadata:

      labels:

        task: monitoring

        k8s-app: influxdb

    spec:

      containers:

      - name: influxdb

        image: gcr.io/google_containers/heapster-influxdb-amd64:v1.3.3

        volumeMounts:

        - mountPath: /data

          name: influxdb-storage

      volumes:

      - name: influxdb-storage

        emptyDir: {}

---

apiVersion: v1

kind: Service

metadata:

  labels:

    task: monitoring

    # For use as a Cluster add-on (https://github.com/kubernetes/kubernetes/tree/master/cluster/addons)

    # If you are NOT using this as an addon, you should comment out this line.

    kubernetes.io/cluster-service: 'true'

    kubernetes.io/name: monitoring-influxdb

  name: monitoring-influxdb

  namespace: kube-system

spec:

  ports:

  - port: 8086

    targetPort: 8086

  selector:

    k8s-app: influxdb

EOF