需求：
K8S中的服务不可用的时候（假定k8s的某个服务出现完全不可用，理论上k8s本身会避免这个问题的出现，这里姑且认为客户需求是合理的），需要为通过CIS已经发布的VS 提供备份访问，及该vs将自动把业务导向其它静态vm提供的服务。
Solution：
需借助BIGIP AS3来实现该需求。即CIS通过部署AS3的方式来部署服务，通过在AS3里直接配置静态的低优先级组member来实现备份，一个例子如下：
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 |
[root@k8s-master f5-k8s]# cat f5-vs-as3.yaml kind: ConfigMap apiVersion: v1 metadata: name: nginx-as3 labels: f5type: virtual-server as3: "true" data: template: | { "class": "AS3", "action": "deploy", "persist": true, "declaration": { "class": "ADC", "schemaVersion": "3.10.0", "id": "123abc", "label": "k8s", "remark": "HTTPS with predictive-node pool and connection limit", "k8sas3": { "class": "Tenant", "nginxservice": { "class": "Application", "template": "https", "serviceMain": { "class": "Service_HTTPS", "virtualAddresses": [ "192.0.2.11" ], "pool": "web_pool", "serverTLS": "webtls" }, "web_pool": { "class": "Pool", "loadBalancingMode": "predictive-node", "monitors": [ "http" ], "members": [ { "servicePort": 80, "serverAddresses": [], "priorityGroup": 5, "connectionLimit": 10 }, { "servicePort": 88, "serverAddresses": [ "2.54.40.3", "6.65.22.2" ], "priorityGroup": 0, "connectionLimit": 20 } ] }, "webtls": { "class": "TLS_Server", "certificates": [{ "certificate": "webcert" }] }, "webcert": { "class": "Certificate", "remark": "in practice we recommend using a passphrase", "certificate": "-----BEGIN CERTIFICATE-----\nMIICnDCCAgWgAwIBAgIJAJ5n2b0OCEjwMA0GCSqGSIb3DQEBCwUAMGcxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQHDAdTZWF0dGxlMRQwEgYDVQQKDAtmNV9OZXR3b3JrczEbMBkGA1UEAwwSc2FtcGxlLmV4YW1wbGUubmV0MB4XDTE3MTEyNjE5NTAyNFoXDTE4MDIyNTE5NTAyNFowZzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCldhc2hpbmd0b24xEDAOBgNVBAcMB1NlYXR0bGUxFDASBgNVBAoMC2Y1X05ldHdvcmtzMRswGQYDVQQDDBJzYW1wbGUuZXhhbXBsZS5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBALEsuXmSXVQpYjrZPW+WiTBjn491mwZYT7Q92V1HlSBtM6WdWlK1aZN5sovfKtOX7Yrm8xa+e4o/zJ2QYLyyv5O+t2EGN/4qUEjEAPY9mwJdfzRQy6Hyzm84J0QkTuUJ/EjNuPji3D0QJRALUTzu1UqqDCEtiN9OGyXEkh7uvb7BAgMBAAGjUDBOMB0GA1UdDgQWBBSVHPNrGWrjWyZvckQxFYWO59FRFjAfBgNVHSMEGDAWgBSVHPNrGWrjWyZvckQxFYWO59FRFjAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GBAJeJ9SEckEwPhkXOm+IuqfbUS/RcziifBCTmVyE+Fa/j9pKSYTgiEBNdbJeBEa+gPMlQtbV7Y2dy8TKx/8axVBHiXC5geDML7caxOrAyHYBpnx690xJTh5OIORBBM/a/NvaR+P3CoVebr/NPRh9oRNxnntnqvqD7SW0U3ZPe3tJc\n-----END CERTIFICATE-----", "privateKey": "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-256-CBC,D8FFCE6B255601587CB54EC29B737D31\n\nkv4Fc3Jn0Ujkj0yRjt+gQQfBLSNF2aRLUENXnlr7Xpzqu0Ahr3jS1bAAnd8IWnsR\nyILqVmKsYF2DoHh0tWiEAQ7/y/fe5DTFhK7N4Wml6kp2yVMkP6KC4ssyYPw27kjK\nDBwBZ5O8Ioej08A5sgsLCmglbmtSPHJUn14pQnMTmLOpEtOsu6S+2ibPgSNpdg0b\nCAJNG/KHe+Vkx59qNDyDeKb7FZOlsX30+y67zUq9GQqJEDuysPJ2BUNP0IJXAjst\nFIt1qNoZew+5KDYs7u/lPxcMGTirUhgI84Jy4WcDvSOsP/tKlxj04TbIE3epmSKy\n+TihHkwY7ngIGtcm3Sfqk5jz2RXoj1/Ac3SW8kVTYaOUogBhn7zAq4Wju6Et4hQG\nRGapsJp1aCeZ/a4RCDTxspcKoMaRa97/URQb0hBRGx3DGUhzpmX9zl7JI2Xa5D3R\nmdBXtjLKYJTdIMdd27prBEKhMUpae2rz5Mw4J907wZeBq/wu+zp8LAnecfTe2nGY\nE32x1U7gSEdYOGqnwxsOexb1jKgCa67Nw9TmcMPV8zmH7R9qdvgxAbAtwBl1F9OS\nfcGaC7epf1AjJLtaX7krWmzgASHl28Ynh9lmGMdv+5QYMZvKG0LOg/n3m8uJ6sKy\nIzzvaJswwn0j5P5+czyoV5CvvdCfKnNb+3jUEN8I0PPwjBGKr4B1ojwhogTM248V\nHR69D6TxFVMfGpyJhCPkbGEGbpEpcffpgKuC/mEtMqyDQXJNaV5HO6HgAJ9F1P6v\n5ehHHTMRvzCCFiwndHdlMXUjqSNjww6me6dr6LiAPbejdzhL2vWx1YqebOcwQx3G\n-----END RSA PRIVATE KEY-----", "passphrase": { "ciphertext": "ZjVmNQ==", "protected": "eyJhbGciOiJkaXIiLCJlbmMiOiJub25lIn0" } } } } } } |
上述配置中的pool members部分是重点，静态添加低优先级组member，高优先级组由CIS自动化发现
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 |
"members": [ { "servicePort": 80, "serverAddresses": [], "priorityGroup": 5, "connectionLimit": 10 }, { "servicePort": 88, "serverAddresses": [ "2.54.40.3", "6.65.22.2" ], "priorityGroup": 0, "connectionLimit": 20 } ] |
最终在F5上产生的配置效果如下：
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 |
root@(v13-common)(cfg-sync Not All Devices Synced)(Active)(/k8sas3/nginxservice)(tmos)# list ltm pool web_pool ltm pool web_pool { load-balancing-mode predictive-node members { /k8sas3/10.244.0.182:http { address 10.244.0.182 connection-limit 10 priority-group 5 session monitor-enabled state up metadata { source { value declaration } } } /k8sas3/10.244.1.129:http { address 10.244.1.129 connection-limit 10 priority-group 5 session monitor-enabled state up metadata { source { value declaration } } } /k8sas3/2.54.40.3:kerberos { address 2.54.40.3 connection-limit 20 session monitor-enabled state down metadata { source { value declaration } } } /k8sas3/6.65.22.2:kerberos { address 6.65.22.2 connection-limit 20 session monitor-enabled state down metadata { source { value declaration } } } } min-active-members 1 monitor min 1 of { /Common/http } partition k8sas3 } |
测试：
如果删除相关svc：
[root@k8s-master f5-k8s]# kubectl delete -f nginx-deploy-svc.yaml
CIS将只留下静态部分的pool member
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
root@(v13-common)(cfg-sync Not All Devices Synced)(Active)(/k8sas3/nginxservice)(tmos)# list ltm pool web_pool ltm pool web_pool { load-balancing-mode predictive-node members { /k8sas3/2.54.40.3:kerberos { address 2.54.40.3 connection-limit 20 session monitor-enabled state down metadata { source { value declaration } } } /k8sas3/6.65.22.2:kerberos { address 6.65.22.2 connection-limit 20 session monitor-enabled state down metadata { source { value declaration } } } } min-active-members 1 monitor min 1 of { /Common/http } partition k8sas3 } |
恢复相关svc：
[root@k8s-master f5-k8s][root@k8s-master f5-k8s]# kubectl create -f nginx-deploy-svc.yaml
pod再次被自动添加
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 |
root@(v13-common)(cfg-sync Not All Devices Synced)(Active)(/k8sas3/nginxservice)(tmos)# list ltm pool web_pool ltm pool web_pool { load-balancing-mode predictive-node members { /k8sas3/10.244.0.182:http { address 10.244.0.182 connection-limit 10 priority-group 5 session monitor-enabled state up metadata { source { value declaration } } } /k8sas3/10.244.1.129:http { address 10.244.1.129 connection-limit 10 priority-group 5 session monitor-enabled state up metadata { source { value declaration } } } /k8sas3/2.54.40.3:kerberos { address 2.54.40.3 connection-limit 20 session monitor-enabled state down metadata { source { value declaration } } } /k8sas3/6.65.22.2:kerberos { address 6.65.22.2 connection-limit 20 session monitor-enabled state down metadata { source { value declaration } } } } min-active-members 1 monitor min 1 of { /Common/http } partition k8sas3 } |
其它：
此方法需要用户采用AS3来进行配置，用户应充分理解AS3本身的特性和用法，以及了解AS3在与CIS配合时的一些限制。具体可参考以下内容：
https://clouddocs.f5.com/products/extensions/f5-appsvcs-extension/latest/
https://clouddocs.f5.com/containers/v2/kubernetes/kctlr-k8s-as3.html
文章评论