[title]证书准备工作[/title]
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 |
openssl genrsa -out ca.key 2048 openssl req -x509 -nodes -new -key ca.key -subj "/CN=k8s.root.ca" -days 7200 -out ca.crt openssl x509 -in ca.crt -text -noout openssl genrsa -out apiserver.key 2048 openssl req -new -key apiserver.key -subj "/CN=172.16.199.17" -config api_openssl.conf -out apiserver.csr vi api_openssl.conf,内容如下: **** [root@docker1 ssl]# more api_openssl.conf [req] req_extensions = v3_req distinguished_name = req_distinguished_name [req_distinguished_name] [ v3_req ] basicConstraints = CA:FALSE keyUsage = nonRepudiation, digitalSignature, keyEncipherment subjectAltName = @alt_names [alt_names] DNS.1 = kubernetes DNS.2 = kubernetes.default DNS.3 = kubernetes.default.svc DNS.4 = kubernetes.default.svc.cluster.local IP.1 = 169.169.0.1 IP.2 = 172.16.199.17 ***** openssl x509 -req -in apiserver.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 7200 -extensions v3_req -extfile api_openssl.conf -out apiserver.crt openssl genrsa -out kubeproxy.key 2048 vi kube-proxy-openssl.cnf ，内容如下 **** [root@docker1 ssl]# more kube-proxy-openssl.cnf [req] req_extensions = v3_req distinguished_name = req_distinguished_name [req_distinguished_name] [ v3_req ] basicConstraints = CA:FALSE keyUsage = nonRepudiation, digitalSignature, keyEncipherment subjectAltName = @alt_names [alt_names] IP.1 = 172.16.199.27 IP.2 = 172.16.199.37 ***** openssl req -new -key kubeproxy.key -subj "/CN=system:kube-proxy" -config kube-proxy-openssl.cnf -out kubeproxy.csr openssl x509 -req -in kubeproxy.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 7200 -extensions v3_req -extfile kube-proxy-openssl.cnf -out kubeproxy.crt scp ca.crt kubeproxy.crt kubeproxy.key 172.16.199.27:/etc/kubernetes/ scp ca.crt kubeproxy.crt kubeproxy.key 172.16.199.37:/etc/kubernetes/ ---- 创建bootstraping token文件： export BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ') cat > token.csv < |
[title]所有服务的配置脚本[/title]
apiserver
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 |
#!/bin/bash # Copyright 2014 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. MASTER_ADDRESS=${1:-"172.16.199.17"} ETCD_SERVERS=${2:-"http://172.16.199.17:2379"} SERVICE_CLUSTER_IP_RANGE=${3:-"169.169.0.0/16"} ADMISSION_CONTROL=${4:-"NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"} mkdir -p /etc/kubernetes cat <<EOF >/etc/kubernetes/kube-apiserver # --alsologtostderr=true: log to standard error as well as log file KUBE_ALSO_LOGTOSTDERR="--alsologtostderr=true" # --log-dir KUBE_LOG_DIR="--log-dir=/var/log/kubernetes" # --v=0: log level for V logs KUBE_LOG_LEVEL="--v=2" #stderrthreshold: only show log that over the threshold to stderr STDERR_THRESHOLD="--stderrthreshold=2" # --etcd-servers=[]: List of etcd servers to watch (http://ip:port), # comma separated. Mutually exclusive with -etcd-config KUBE_ETCD_SERVERS="--etcd-servers=${ETCD_SERVERS}" # --etcd-cafile="": SSL Certificate Authority file used to secure etcd communication. #KUBE_ETCD_CAFILE="--etcd-cafile=/srv/kubernetes/etcd/ca.pem" # --etcd-certfile="": SSL certification file used to secure etcd communication. #KUBE_ETCD_CERTFILE="--etcd-certfile=/srv/kubernetes/etcd/client.pem" # --etcd-keyfile="": key file used to secure etcd communication. #KUBE_ETCD_KEYFILE="--etcd-keyfile=/srv/kubernetes/etcd/client-key.pem" # --insecure-bind-address=127.0.0.1: The IP address on which to serve the --insecure-port. KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0" # --insecure-port=8080: The port on which to serve unsecured, unauthenticated access. KUBE_API_PORT="--insecure-port=8080" # --kubelet-port=10250: Kubelet port NODE_PORT="--kubelet-port=10250" # --advertise-address=<nil>: The IP address on which to advertise # the apiserver to members of the cluster. KUBE_ADVERTISE_ADDR="--advertise-address=${MASTER_ADDRESS}" # --allow-privileged=false: If true, allow privileged containers. KUBE_ALLOW_PRIV="--allow-privileged=true" # --service-cluster-ip-range=<nil>: A CIDR notation IP range from which to assign service cluster IPs. # This must not overlap with any IP ranges assigned to nodes for pods. KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" # --admission-control="AlwaysAdmit": Ordered list of plug-ins # to do admission control of resources into cluster. # Comma-delimited list of: # LimitRanger, AlwaysDeny, SecurityContextDeny, NamespaceExists, # NamespaceLifecycle, NamespaceAutoProvision, AlwaysAdmit, # ServiceAccount, DefaultStorageClass, DefaultTolerationSeconds, ResourceQuota KUBE_ADMISSION_CONTROL="--admission-control=${ADMISSION_CONTROL}" # --client-ca-file="": If set, any request presenting a client certificate signed # by one of the authorities in the client-ca-file is authenticated with an identity # corresponding to the CommonName of the client certificate. KUBE_API_CLIENT_CA_FILE="--client-ca-file=/etc/kubernetes/ca.crt" # --tls-cert-file="": File containing x509 Certificate for HTTPS. (CA cert, if any, # concatenated after server cert). If HTTPS serving is enabled, and --tls-cert-file # and --tls-private-key-file are not provided, a self-signed certificate and key are # generated for the public address and saved to /var/run/kubernetes. KUBE_API_TLS_CERT_FILE="--tls-cert-file=/etc/kubernetes/apiserver.crt" # --tls-private-key-file="": File containing x509 private key matching --tls-cert-file. KUBE_API_TLS_PRIVATE_KEY_FILE="--tls-private-key-file=/etc/kubernetes/apiserver.key" ##enable RBAC add-by-myf5.net KUBE_RBAC="--authorization-mode=RBAC" ##OTHERS KUBE_RUN_TIME="--runtime-config=rbac.authorization.k8s.io/v1beta1" KUBE_KUBELET_EN="--kubelet-https=true" KUBE_BS_TOKEN="--experimental-bootstrap-token-auth" KUBE_TOKEN_FILE="--token-auth-file=/etc/kubernetes/token.csv" KUBE_SA_KEY="--service-account-key-file=/etc/kubernetes/ca.key" EOF KUBE_APISERVER_OPTS=" \${KUBE_ALSO_LOGTOSTDERR} \\ \${STDERR_THRESHOLD} \\ \${KUBE_LOG_DIR} \\ \${KUBE_LOG_LEVEL} \\ \${KUBE_ETCD_SERVERS} \\ \${KUBE_API_ADDRESS} \\ \${KUBE_API_PORT} \\ \${NODE_PORT} \\ \${KUBE_ADVERTISE_ADDR} \\ \${KUBE_ALLOW_PRIV} \\ \${KUBE_SERVICE_ADDRESSES} \\ \${KUBE_API_CLIENT_CA_FILE} \\ \${KUBE_API_TLS_CERT_FILE} \\ \${KUBE_API_TLS_PRIVATE_KEY_FILE} \\ \${KUBE_ADMISSION_CONTROL} \\ \${KUBE_RBAC} \\ \${KUBE_KUBELET_EN} \\ \${KUBE_RUN_TIME} \\ \${KUBE_BS_TOKEN} \\ \${KUBE_TOKEN_FILE} \\ \${KUBE_SA_KEY}" cat <<EOF >/usr/lib/systemd/system/kube-apiserver.service [Unit] Description=Kubernetes API Server Documentation=https://github.com/kubernetes/kubernetes [Service] EnvironmentFile=-/etc/kubernetes/kube-apiserver ExecStart=/usr/bin/kube-apiserver ${KUBE_APISERVER_OPTS} Restart=on-failure [Install] WantedBy=multi-user.target EOF systemctl daemon-reload systemctl enable kube-apiserver systemctl restart kube-apiserver |
controller-manger
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 |
#!/bin/bash # Copyright 2014 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. MASTER_ADDRESS=${1:-"172.16.199.17"} cat <<EOF >/etc/kubernetes/kube-controller-manager KUBE_LOGTOSTDERR="--logtostderr=false" KUBE_LOG_DIR="--log-dir=/var/log/kubernetes" KUBE_LOG_LEVEL="--v=4" KUBE_MASTER="--master=${MASTER_ADDRESS}:8080" KUBE_CLUSTER_CIDR="--cluster-cidr=10.1.0.0/16" KUBE_SVC_CLUSTER_RANGE="--service-cluster-ip-range=169.169.0.0/16" # --root-ca-file="": If set, this root certificate authority will be included in # service account's token secret. This must be a valid PEM-encoded CA bundle. KUBE_CONTROLLER_MANAGER_ROOT_CA_FILE="--root-ca-file=/etc/kubernetes/ca.crt" # --service-account-private-key-file="": Filename containing a PEM-encoded private # RSA key used to sign service account tokens. KUBE_CONTROLLER_MANAGER_SERVICE_ACCOUNT_PRIVATE_KEY_FILE="--service-account-private-key-file=/etc/kubernetes/ca.key" #Used for sign cert for kubelet KUBE_SIGN_CA="--cluster-signing-cert-file=/etc/kubernetes/ca.crt" KUBE_SIGN_CA_KEY="--cluster-signing-key-file=/etc/kubernetes/ca.key" KUBE_CLUSTER_NAME="--cluster-name=kubernetes" # --leader-elect KUBE_LEADER_ELECT="--leader-elect=false" EOF KUBE_CONTROLLER_MANAGER_OPTS=" \${KUBE_LOGTOSTDERR} \\ \${KUBE_LOG_DIR} \\ \${KUBE_LOG_LEVEL} \\ \${KUBE_MASTER} \\ \${KUBE_CLUSTER_CIDR} \\ \${KUBE_CLUSTER_NAME} \\ \${KUBE_SVC_CLUSTER_RANGE} \\ \${KUBE_CONTROLLER_MANAGER_ROOT_CA_FILE} \\ \${KUBE_CONTROLLER_MANAGER_SERVICE_ACCOUNT_PRIVATE_KEY_FILE} \\ \${KUBE_SIGN_CA} \\ \${KUBE_SIGN_CA_KEY} \\ \${KUBE_LEADER_ELECT}" cat <<EOF >/usr/lib/systemd/system/kube-controller-manager.service [Unit] Description=Kubernetes Controller Manager Documentation=https://github.com/kubernetes/kubernetes After=kube-apiserver.service After=network.target Requires=kube-apiserver.service [Service] EnvironmentFile=-/etc/kubernetes/kube-controller-manager ExecStart=/usr/bin/kube-controller-manager ${KUBE_CONTROLLER_MANAGER_OPTS} Restart=on-failure [Install] WantedBy=multi-user.target EOF systemctl daemon-reload systemctl enable kube-controller-manager systemctl restart kube-controller-manager |
scheduler
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
#!/bin/bash # Copyright 2014 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. MASTER_ADDRESS=${1:-"172.16.199.17"} cat <<EOF >/etc/kubernetes/kube-scheduler ### # kubernetes scheduler config # --logtostderr=true: log to standard error instead of files KUBE_LOGTOSTDERR="--logtostderr=false" # Log to file KUBE_LOG_DIR="--log-dir=/var/log/kubernetes" # --v=0: log level for V logs KUBE_LOG_LEVEL="--v=4" KUBE_MASTER="--master=${MASTER_ADDRESS}:8080" # --leader-elect KUBE_LEADER_ELECT="--leader-elect=false" # Add your own! KUBE_SCHEDULER_ARGS="" EOF KUBE_SCHEDULER_OPTS=" \${KUBE_LOGTOSTDERR} \\ \${KUBE_LOG_DIR} \\ \${KUBE_LOG_LEVEL} \\ \${KUBE_MASTER} \\ \${KUBE_LEADER_ELECT} \\ \${KUBE_SCHEDULER_ARGS}" cat <<EOF >/usr/lib/systemd/system/kube-scheduler.service [Unit] Description=Kubernetes Scheduler Documentation=https://github.com/kubernetes/kubernetes After=kube-apiserver.service Requires=kube-apiserver.service [Service] EnvironmentFile=-/etc/kubernetes/kube-scheduler ExecStart=/usr/bin/kube-scheduler ${KUBE_SCHEDULER_OPTS} Restart=on-failure [Install] WantedBy=multi-user.target EOF systemctl daemon-reload systemctl enable kube-scheduler systemctl restart kube-scheduler |
Node节点：
kubelet
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 |
#!/bin/bash # Copyright 2014 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. MASTER_ADDRESS=${1:-"172.16.199.17"} NODE_ADDRESS=${2:-"172.16.199.27"} DNS_SERVER_IP=${3:-"169.169.0.53"} DNS_DOMAIN=${4:-"cluster.local"} mkdir -p /etc/kubernetes cat <<EOF >/etc/kubernetes/kubelet # --logtostderr=true: log to standard error instead of files KUBE_LOGTOSTDERR="--logtostderr=true" #LOG TO FILE KUBE_LOG_DIR="--log-dir=/var/log/kubernetes" # --v=0: log level for V logs KUBE_LOG_LEVEL="--v=4" # --address=0.0.0.0: The IP address for the Kubelet to serve on (set to 0.0.0.0 for all interfaces) NODE_ADDRESS="--address=${NODE_ADDRESS}" # --port=10250: The port for the Kubelet to serve on. Note that "kubectl logs" will not work if you set this flag. NODE_PORT="" # --hostname-override="": If non-empty, will use this string as identification instead of the actual hostname. NODE_HOSTNAME="--hostname-override=${NODE_ADDRESS}" #--api-servers=[]: List of Kubernetes API servers for publishing events, # and reading pods and services. (ip:port), comma separated. KUBELET_API_SERVER="--api-servers=http://${MASTER_ADDRESS}:8080" # --allow-privileged=false: If true, allow containers to request privileged mode. [default=false] KUBE_ALLOW_PRIV="--allow-privileged=false" # DNS info KUBELET__DNS_IP="--cluster-dns=${DNS_SERVER_IP}" KUBELET_DNS_DOMAIN="--cluster-domain=${DNS_DOMAIN}" # Add your own! KUBELET_ARGS=“--cgroup-driver=systemd --experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig --kubeconfig=/etc/kubernetes/kubelet.kubeconfig --require-kubeconfig --cert-dir=/etc/kubernetes/ssl" EOF KUBE_PROXY_OPTS=" \${KUBE_LOGTOSTDERR} \\ \${KUBE_LOG_DIR} \\ \${KUBE_LOG_LEVEL} \\ \${NODE_ADDRESS} \\ \${NODE_PORT} \\ \${NODE_HOSTNAME} \\ \${KUBELET_API_SERVER} \\ \${KUBE_ALLOW_PRIV} \\ \${KUBELET__DNS_IP} \\ \${KUBELET_DNS_DOMAIN} \\ \${KUBELET_ARGS}" cat <<EOF >/usr/lib/systemd/system/kubelet.service [Unit] Description=Kubernetes Kubelet After=docker.service Requires=docker.service [Service] EnvironmentFile=-/etc/kubernetes/kubelet ExecStart=/usr/bin/kubelet ${KUBE_PROXY_OPTS} Restart=on-failure KillMode=process [Install] WantedBy=multi-user.target EOF systemctl daemon-reload systemctl enable kubelet systemctl restart kubelet |
kubeproxy
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 |
#!/bin/bash # Copyright 2014 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. MASTER_ADDRESS=${1:-"172.16.199.17"} NODE_ADDRESS=${2:-"172.16.199.27"} DNS_SERVER_IP=${3:-"169.169.0.53"} DNS_DOMAIN=${4:-"cluster.local"} mkdir -p /etc/kubernetes cat <<EOF >/etc/kubernetes/kubelet # --logtostderr=true: log to standard error instead of files KUBE_LOGTOSTDERR="--logtostderr=true" #LOG TO FILE KUBE_LOG_DIR="--log-dir=/var/log/kubernetes" # --v=0: log level for V logs KUBE_LOG_LEVEL="--v=4" # --address=0.0.0.0: The IP address for the Kubelet to serve on (set to 0.0.0.0 for all interfaces) NODE_ADDRESS="--address=${NODE_ADDRESS}" # --port=10250: The port for the Kubelet to serve on. Note that "kubectl logs" will not work if you set this flag. NODE_PORT="" # --hostname-override="": If non-empty, will use this string as identification instead of the actual hostname. NODE_HOSTNAME="--hostname-override=${NODE_ADDRESS}" #--api-servers=[]: List of Kubernetes API servers for publishing events, # and reading pods and services. (ip:port), comma separated. KUBELET_API_SERVER="--api-servers=http://${MASTER_ADDRESS}:8080" # --allow-privileged=false: If true, allow containers to request privileged mode. [default=false] KUBE_ALLOW_PRIV="--allow-privileged=false" # DNS info KUBELET__DNS_IP="--cluster-dns=${DNS_SERVER_IP}" KUBELET_DNS_DOMAIN="--cluster-domain=${DNS_DOMAIN}" # Add your own! KUBELET_ARGS=“--cgroup-driver=systemd --experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig --kubeconfig=/etc/kubernetes/kubelet.kubeconfig --require-kubeconfig --cert-dir=/etc/kubernetes/ssl" EOF KUBE_PROXY_OPTS=" \${KUBE_LOGTOSTDERR} \\ \${KUBE_LOG_DIR} \\ \${KUBE_LOG_LEVEL} \\ \${NODE_ADDRESS} \\ \${NODE_PORT} \\ \${NODE_HOSTNAME} \\ \${KUBELET_API_SERVER} \\ \${KUBE_ALLOW_PRIV} \\ \${KUBELET__DNS_IP} \\ \${KUBELET_DNS_DOMAIN} \\ \${KUBELET_ARGS}" cat <<EOF >/usr/lib/systemd/system/kubelet.service [Unit] Description=Kubernetes Kubelet After=docker.service Requires=docker.service [Service] EnvironmentFile=-/etc/kubernetes/kubelet ExecStart=/usr/bin/kubelet ${KUBE_PROXY_OPTS} Restart=on-failure KillMode=process [Install] WantedBy=multi-user.target EOF systemctl daemon-reload systemctl enable kubelet systemctl restart kubelet |
上述配置主要是希望实现kubelet能够通过bootstrap自动获取到证书，并通过证书与APIserver通信，从实际结果看，kubelet没有自动申请csr，在master上无法get到相关csr请求。有待查找问题。但整体配置不影响使用。
另，节点部分看上去其实可以保持使用非SSL配置方式，即保留Kubernetes 1.6.7 手工安装里的node配置
文章评论