K8S，V1.6.7 双向TLS配置

Warning: Array to string conversion in /home/wwwroot/myf5/wp-content/plugins/wpjam-basic/public/wpjam-utils.php on line 496

[title]证书准备工作[/title]

openssl  genrsa  -out ca.key 2048
openssl  req -x509 -nodes  -new -key ca.key -subj "/CN=k8s.root.ca" -days 7200 -out ca.crt
openssl x509 -in ca.crt -text -noout


openssl genrsa -out apiserver.key 2048	
openssl req -new -key apiserver.key  -subj "/CN=172.16.199.17" -config api_openssl.conf -out apiserver.csr
vi api_openssl.conf,内容如下:
****
[root@docker1 ssl]# more api_openssl.conf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
DNS.1 = kubernetes
DNS.2 = kubernetes.default
DNS.3 = kubernetes.default.svc
DNS.4 = kubernetes.default.svc.cluster.local
IP.1 = 169.169.0.1
IP.2 = 172.16.199.17
*****

openssl x509 -req -in apiserver.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 7200 -extensions v3_req -extfile api_openssl.conf -out apiserver.crt


openssl genrsa -out kubeproxy.key 2048
vi kube-proxy-openssl.cnf ，内容如下
****
[root@docker1 ssl]# more kube-proxy-openssl.cnf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
IP.1 = 172.16.199.27
IP.2 = 172.16.199.37
*****

openssl req -new -key kubeproxy.key  -subj "/CN=system:kube-proxy" -config kube-proxy-openssl.cnf -out kubeproxy.csr
openssl x509 -req -in kubeproxy.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 7200 -extensions v3_req -extfile kube-proxy-openssl.cnf -out kubeproxy.crt


scp ca.crt kubeproxy.crt kubeproxy.key 172.16.199.27:/etc/kubernetes/
scp ca.crt kubeproxy.crt kubeproxy.key 172.16.199.37:/etc/kubernetes/
----
创建bootstraping token文件：
export BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')
cat > token.csv <

[title]所有服务的配置脚本[/title]

apiserver

#!/bin/bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


MASTER_ADDRESS=${1:-"172.16.199.17"}
ETCD_SERVERS=${2:-"http://172.16.199.17:2379"}
SERVICE_CLUSTER_IP_RANGE=${3:-"169.169.0.0/16"}
ADMISSION_CONTROL=${4:-"NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"}

mkdir -p /etc/kubernetes

cat <<EOF >/etc/kubernetes/kube-apiserver
# --alsologtostderr=true: log to standard error as well as log file
KUBE_ALSO_LOGTOSTDERR="--alsologtostderr=true"
# --log-dir
KUBE_LOG_DIR="--log-dir=/var/log/kubernetes"
# --v=0: log level for V logs
KUBE_LOG_LEVEL="--v=2"
#stderrthreshold: only show log that over the threshold to stderr
STDERR_THRESHOLD="--stderrthreshold=2"
# --etcd-servers=[]: List of etcd servers to watch (http://ip:port),
# comma separated. Mutually exclusive with -etcd-config
KUBE_ETCD_SERVERS="--etcd-servers=${ETCD_SERVERS}"
# --etcd-cafile="": SSL Certificate Authority file used to secure etcd communication.
#KUBE_ETCD_CAFILE="--etcd-cafile=/srv/kubernetes/etcd/ca.pem"
# --etcd-certfile="": SSL certification file used to secure etcd communication.
#KUBE_ETCD_CERTFILE="--etcd-certfile=/srv/kubernetes/etcd/client.pem"
# --etcd-keyfile="": key file used to secure etcd communication.
#KUBE_ETCD_KEYFILE="--etcd-keyfile=/srv/kubernetes/etcd/client-key.pem"
# --insecure-bind-address=127.0.0.1: The IP address on which to serve the --insecure-port.
KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0"
# --insecure-port=8080: The port on which to serve unsecured, unauthenticated access.
KUBE_API_PORT="--insecure-port=8080"
# --kubelet-port=10250: Kubelet port
NODE_PORT="--kubelet-port=10250"
# --advertise-address=<nil>: The IP address on which to advertise
# the apiserver to members of the cluster.
KUBE_ADVERTISE_ADDR="--advertise-address=${MASTER_ADDRESS}"
# --allow-privileged=false: If true, allow privileged containers.
KUBE_ALLOW_PRIV="--allow-privileged=true"
# --service-cluster-ip-range=<nil>: A CIDR notation IP range from which to assign service cluster IPs.
# This must not overlap with any IP ranges assigned to nodes for pods.
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
# --admission-control="AlwaysAdmit": Ordered list of plug-ins
# to do admission control of resources into cluster.
# Comma-delimited list of:
#   LimitRanger, AlwaysDeny, SecurityContextDeny, NamespaceExists,
#   NamespaceLifecycle, NamespaceAutoProvision, AlwaysAdmit,
#   ServiceAccount, DefaultStorageClass, DefaultTolerationSeconds, ResourceQuota
KUBE_ADMISSION_CONTROL="--admission-control=${ADMISSION_CONTROL}"

# --client-ca-file="": If set, any request presenting a client certificate signed
# by one of the authorities in the client-ca-file is authenticated with an identity
# corresponding to the CommonName of the client certificate.
KUBE_API_CLIENT_CA_FILE="--client-ca-file=/etc/kubernetes/ca.crt"

# --tls-cert-file="": File containing x509 Certificate for HTTPS.  (CA cert, if any,
# concatenated after server cert). If HTTPS serving is enabled, and --tls-cert-file
# and --tls-private-key-file are not provided, a self-signed certificate and key are
# generated for the public address and saved to /var/run/kubernetes.
KUBE_API_TLS_CERT_FILE="--tls-cert-file=/etc/kubernetes/apiserver.crt"

# --tls-private-key-file="": File containing x509 private key matching --tls-cert-file.
KUBE_API_TLS_PRIVATE_KEY_FILE="--tls-private-key-file=/etc/kubernetes/apiserver.key"

##enable RBAC add-by-myf5.net
KUBE_RBAC="--authorization-mode=RBAC"

##OTHERS
KUBE_RUN_TIME="--runtime-config=rbac.authorization.k8s.io/v1beta1"
KUBE_KUBELET_EN="--kubelet-https=true"
KUBE_BS_TOKEN="--experimental-bootstrap-token-auth"
KUBE_TOKEN_FILE="--token-auth-file=/etc/kubernetes/token.csv"
KUBE_SA_KEY="--service-account-key-file=/etc/kubernetes/ca.key"
EOF

KUBE_APISERVER_OPTS="   \${KUBE_ALSO_LOGTOSTDERR}    \\
                        \${STDERR_THRESHOLD}         \\
                        \${KUBE_LOG_DIR}             \\
                        \${KUBE_LOG_LEVEL}           \\
                        \${KUBE_ETCD_SERVERS}        \\
                        \${KUBE_API_ADDRESS}         \\
                        \${KUBE_API_PORT}            \\
                        \${NODE_PORT}                \\
                        \${KUBE_ADVERTISE_ADDR}      \\
                        \${KUBE_ALLOW_PRIV}          \\
                        \${KUBE_SERVICE_ADDRESSES}   \\
                        \${KUBE_API_CLIENT_CA_FILE}  \\
                        \${KUBE_API_TLS_CERT_FILE}   \\
                        \${KUBE_API_TLS_PRIVATE_KEY_FILE} \\
                        \${KUBE_ADMISSION_CONTROL} \\
                        \${KUBE_RBAC} \\
                        \${KUBE_KUBELET_EN} \\
                        \${KUBE_RUN_TIME} \\
                        \${KUBE_BS_TOKEN} \\
                        \${KUBE_TOKEN_FILE} \\
                        \${KUBE_SA_KEY}"


cat <<EOF >/usr/lib/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=-/etc/kubernetes/kube-apiserver
ExecStart=/usr/bin/kube-apiserver ${KUBE_APISERVER_OPTS}
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable kube-apiserver
systemctl restart kube-apiserver

controller-manger

#!/bin/bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


MASTER_ADDRESS=${1:-"172.16.199.17"}

cat <<EOF >/etc/kubernetes/kube-controller-manager
KUBE_LOGTOSTDERR="--logtostderr=false"
KUBE_LOG_DIR="--log-dir=/var/log/kubernetes"
KUBE_LOG_LEVEL="--v=4"
KUBE_MASTER="--master=${MASTER_ADDRESS}:8080"

KUBE_CLUSTER_CIDR="--cluster-cidr=10.1.0.0/16"
KUBE_SVC_CLUSTER_RANGE="--service-cluster-ip-range=169.169.0.0/16"
# --root-ca-file="": If set, this root certificate authority will be included in
# service account's token secret. This must be a valid PEM-encoded CA bundle.
KUBE_CONTROLLER_MANAGER_ROOT_CA_FILE="--root-ca-file=/etc/kubernetes/ca.crt"

# --service-account-private-key-file="": Filename containing a PEM-encoded private
# RSA key used to sign service account tokens.
KUBE_CONTROLLER_MANAGER_SERVICE_ACCOUNT_PRIVATE_KEY_FILE="--service-account-private-key-file=/etc/kubernetes/ca.key"

#Used for sign cert for kubelet
KUBE_SIGN_CA="--cluster-signing-cert-file=/etc/kubernetes/ca.crt"
KUBE_SIGN_CA_KEY="--cluster-signing-key-file=/etc/kubernetes/ca.key"

KUBE_CLUSTER_NAME="--cluster-name=kubernetes"

# --leader-elect
KUBE_LEADER_ELECT="--leader-elect=false"
EOF

KUBE_CONTROLLER_MANAGER_OPTS="  \${KUBE_LOGTOSTDERR} \\
                                \${KUBE_LOG_DIR}     \\
                                \${KUBE_LOG_LEVEL}   \\
                                \${KUBE_MASTER}      \\
                                \${KUBE_CLUSTER_CIDR} \\
                                \${KUBE_CLUSTER_NAME} \\
                                \${KUBE_SVC_CLUSTER_RANGE} \\
                                \${KUBE_CONTROLLER_MANAGER_ROOT_CA_FILE} \\
                                \${KUBE_CONTROLLER_MANAGER_SERVICE_ACCOUNT_PRIVATE_KEY_FILE} \\
                                \${KUBE_SIGN_CA} \\
                                \${KUBE_SIGN_CA_KEY} \\
                                \${KUBE_LEADER_ELECT}"

cat <<EOF >/usr/lib/systemd/system/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
After=kube-apiserver.service
After=network.target
Requires=kube-apiserver.service

[Service]
EnvironmentFile=-/etc/kubernetes/kube-controller-manager
ExecStart=/usr/bin/kube-controller-manager ${KUBE_CONTROLLER_MANAGER_OPTS}
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable kube-controller-manager
systemctl restart kube-controller-manager

scheduler

#!/bin/bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


MASTER_ADDRESS=${1:-"172.16.199.17"}

cat <<EOF >/etc/kubernetes/kube-scheduler
###
# kubernetes scheduler config

# --logtostderr=true: log to standard error instead of files
KUBE_LOGTOSTDERR="--logtostderr=false"

# Log to file
KUBE_LOG_DIR="--log-dir=/var/log/kubernetes"

# --v=0: log level for V logs
KUBE_LOG_LEVEL="--v=4"

KUBE_MASTER="--master=${MASTER_ADDRESS}:8080"

# --leader-elect
KUBE_LEADER_ELECT="--leader-elect=false"

# Add your own!
KUBE_SCHEDULER_ARGS=""

EOF

KUBE_SCHEDULER_OPTS="   \${KUBE_LOGTOSTDERR}     \\
                        \${KUBE_LOG_DIR}         \\
                        \${KUBE_LOG_LEVEL}       \\
                        \${KUBE_MASTER}          \\
                        \${KUBE_LEADER_ELECT}    \\
                        \${KUBE_SCHEDULER_ARGS}"

cat <<EOF >/usr/lib/systemd/system/kube-scheduler.service
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
After=kube-apiserver.service
Requires=kube-apiserver.service

[Service]
EnvironmentFile=-/etc/kubernetes/kube-scheduler
ExecStart=/usr/bin/kube-scheduler ${KUBE_SCHEDULER_OPTS}
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable kube-scheduler
systemctl restart kube-scheduler

Node节点：

kubelet

#!/bin/bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


MASTER_ADDRESS=${1:-"172.16.199.17"}
NODE_ADDRESS=${2:-"172.16.199.27"}
DNS_SERVER_IP=${3:-"169.169.0.53"}
DNS_DOMAIN=${4:-"cluster.local"}

mkdir -p /etc/kubernetes

cat <<EOF >/etc/kubernetes/kubelet
# --logtostderr=true: log to standard error instead of files
KUBE_LOGTOSTDERR="--logtostderr=true"
#LOG TO FILE
KUBE_LOG_DIR="--log-dir=/var/log/kubernetes"

#  --v=0: log level for V logs
KUBE_LOG_LEVEL="--v=4"
# --address=0.0.0.0: The IP address for the Kubelet to serve on (set to 0.0.0.0 for all interfaces)
NODE_ADDRESS="--address=${NODE_ADDRESS}"
# --port=10250: The port for the Kubelet to serve on. Note that "kubectl logs" will not work if you set this flag.
NODE_PORT=""
# --hostname-override="": If non-empty, will use this string as identification instead of the actual hostname.
NODE_HOSTNAME="--hostname-override=${NODE_ADDRESS}"
#--api-servers=[]: List of Kubernetes API servers for publishing events,
# and reading pods and services. (ip:port), comma separated.
KUBELET_API_SERVER="--api-servers=http://${MASTER_ADDRESS}:8080"
# --allow-privileged=false: If true, allow containers to request privileged mode. [default=false]
KUBE_ALLOW_PRIV="--allow-privileged=false"
# DNS info
KUBELET__DNS_IP="--cluster-dns=${DNS_SERVER_IP}"
KUBELET_DNS_DOMAIN="--cluster-domain=${DNS_DOMAIN}"
# Add your own!
KUBELET_ARGS=“--cgroup-driver=systemd --experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig --kubeconfig=/etc/kubernetes/kubelet.kubeconfig --require-kubeconfig --cert-dir=/etc/kubernetes/ssl"
EOF

KUBE_PROXY_OPTS="   \${KUBE_LOGTOSTDERR}     \\
                    \${KUBE_LOG_DIR}         \\
                    \${KUBE_LOG_LEVEL}       \\
                    \${NODE_ADDRESS}         \\
                    \${NODE_PORT}            \\
                    \${NODE_HOSTNAME}        \\
                    \${KUBELET_API_SERVER}   \\
                    \${KUBE_ALLOW_PRIV}      \\
                    \${KUBELET__DNS_IP}      \\
                    \${KUBELET_DNS_DOMAIN}      \\
                    \${KUBELET_ARGS}"

cat <<EOF >/usr/lib/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
After=docker.service
Requires=docker.service
[Service]
EnvironmentFile=-/etc/kubernetes/kubelet
ExecStart=/usr/bin/kubelet ${KUBE_PROXY_OPTS}
Restart=on-failure
KillMode=process
[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable kubelet
systemctl restart kubelet

kubeproxy

#!/bin/bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


MASTER_ADDRESS=${1:-"172.16.199.17"}
NODE_ADDRESS=${2:-"172.16.199.27"}
DNS_SERVER_IP=${3:-"169.169.0.53"}
DNS_DOMAIN=${4:-"cluster.local"}

mkdir -p /etc/kubernetes

cat <<EOF >/etc/kubernetes/kubelet
# --logtostderr=true: log to standard error instead of files
KUBE_LOGTOSTDERR="--logtostderr=true"
#LOG TO FILE
KUBE_LOG_DIR="--log-dir=/var/log/kubernetes"

#  --v=0: log level for V logs
KUBE_LOG_LEVEL="--v=4"
# --address=0.0.0.0: The IP address for the Kubelet to serve on (set to 0.0.0.0 for all interfaces)
NODE_ADDRESS="--address=${NODE_ADDRESS}"
# --port=10250: The port for the Kubelet to serve on. Note that "kubectl logs" will not work if you set this flag.
NODE_PORT=""
# --hostname-override="": If non-empty, will use this string as identification instead of the actual hostname.
NODE_HOSTNAME="--hostname-override=${NODE_ADDRESS}"
#--api-servers=[]: List of Kubernetes API servers for publishing events,
# and reading pods and services. (ip:port), comma separated.
KUBELET_API_SERVER="--api-servers=http://${MASTER_ADDRESS}:8080"
# --allow-privileged=false: If true, allow containers to request privileged mode. [default=false]
KUBE_ALLOW_PRIV="--allow-privileged=false"
# DNS info
KUBELET__DNS_IP="--cluster-dns=${DNS_SERVER_IP}"
KUBELET_DNS_DOMAIN="--cluster-domain=${DNS_DOMAIN}"
# Add your own!
KUBELET_ARGS=“--cgroup-driver=systemd --experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig --kubeconfig=/etc/kubernetes/kubelet.kubeconfig --require-kubeconfig --cert-dir=/etc/kubernetes/ssl"
EOF

KUBE_PROXY_OPTS="   \${KUBE_LOGTOSTDERR}     \\
                    \${KUBE_LOG_DIR}         \\
                    \${KUBE_LOG_LEVEL}       \\
                    \${NODE_ADDRESS}         \\
                    \${NODE_PORT}            \\
                    \${NODE_HOSTNAME}        \\
                    \${KUBELET_API_SERVER}   \\
                    \${KUBE_ALLOW_PRIV}      \\
                    \${KUBELET__DNS_IP}      \\
                    \${KUBELET_DNS_DOMAIN}      \\
                    \${KUBELET_ARGS}"

cat <<EOF >/usr/lib/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
After=docker.service
Requires=docker.service
[Service]
EnvironmentFile=-/etc/kubernetes/kubelet
ExecStart=/usr/bin/kubelet ${KUBE_PROXY_OPTS}
Restart=on-failure
KillMode=process
[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable kubelet
systemctl restart kubelet

上述配置主要是希望实现kubelet能够通过bootstrap自动获取到证书，并通过证书与APIserver通信，从实际结果看，kubelet没有自动申请csr，在master上无法get到相关csr请求。有待查找问题。但整体配置不影响使用。

另，节点部分看上去其实可以保持使用非SSL配置方式，即保留Kubernetes 1.6.7 手工安装里的node配置

Array

本作品采用知识共享署名-非商业性使用 4.0 国际许可协议进行许可

K8S，V1.6.7 双向TLS配置

文章评论