GCDW的运行环境k8s已经搭建完成，本文介绍将gcdw部署在k8s上的过程，最终将出现云际WEB管理平台。
目录导航
参考
GCDW技术栈 – kubernets 1.26.0运行环境搭建
GCDW元数据服务FoundationDB的集群模式配置和高可用测试
GCDW技术栈，镜像仓库harbor安装，包括https 服务CA证书的生成(openssl)
上一步
GCDW技术栈 – kubernets 1.26.0运行环境搭建
上传镜像
将gcdw安装包里面的images目录下的tar文件，上传到镜像服务器。
[root@k8s-81 images]# ll
total 3158576
-rw-------. 1 root root 255503360 Jan 4 01:36 gcdw-operator-9.8.0.3.5-202301041735.tar
-rw-------. 1 root root 234355712 Jan 4 01:36 logrotate-202301041735.tar
-rw-------. 1 root root 399340032 Jan 4 01:36 s3-monitor-202301041735.tar
-rw-------. 1 root root 907063808 Jan 6 04:09 saas-backend-9.8.0.3.5-202301061953.tar
-rw-------. 1 root root 155664896 Jan 5 19:23 saas-frontend-9.8.0.3.5-202301061022.tar
-rw-------. 1 root root 108803072 Jan 5 19:23 saas-redis-202301061022.tar
-rw-------. 1 root root 263322112 Jan 5 19:23 saas-vsftpd-202301061022.tar
-rw-------. 1 root root 910307840 Jan 3 23:18 server-9.8.0.3.5-20231513.tar
[root@vm249 images]#先要将镜像加载到docker环境
通过load命令， -i指定文件名，将镜像逐个加载到docker内部。
[root@k8s-81 images]# docker load --help
Usage: docker load [OPTIONS]
Load an image from a tar archive or STDIN
Aliases:
docker image load, docker load
Options:
-i, --input string Read from tar archive file, instead of STDIN
-q, --quiet Suppress the load output
[root@k8s-81 images]# docker load -i saas-backend-9.8.0.3.5-202301061953.tar
d889666f8dfd: Loading layer [==================================================>] 34.53MB/34.53MB
90006678c274: Loading layer [==================================================>] 19.45MB/19.45MB
ce481b589c7b: Loading layer [==================================================>] 136.4MB/136.4MB
5b6d3eb7eb12: Loading layer [==================================================>] 4.052MB/4.052MB
a791e81cb124: Loading layer [==================================================>] 3.072kB/3.072kB
811b32d98e6d: Loading layer [==================================================>] 497.2MB/497.2MB
3d9261d92a9a: Loading layer [==================================================>] 5.205MB/5.205MB
Loaded image: 192.168.8.84/gcdw/clound-database-backend:9.8.0.3.5
[root@vm249 images]# 通过docker images能够看到刚才上传的镜像
给镜像打上tag, 指向镜像服务器
[root@k8s-81 images]# docker tag 192.168.8.84/gcdw/clound-database-backend:9.8.0.3.5 172.16.3.249:8443/gcdw/clound-database-backend:9.8.0.3.5将镜像上传到镜像服务器
[root@k8s-81 images]# docker push 172.16.3.249:8443/gcdw/clound-database-backend:9.8.0.3.5
The push refers to repository [172.16.3.249:8443/gcdw/clound-database-backend]
3d9261d92a9a: Pushed
811b32d98e6d: Pushed
a791e81cb124: Pushed
5b6d3eb7eb12: Pushed
ce481b589c7b: Pushed
90006678c274: Pushed
d889666f8dfd: Pushed
071d8bd76517: Mounted from gcdw/gcdw-operator
9.8.0.3.5: digest: sha256:1dd80b412b5252e66393bc869f3d52aa59bb79044668af85a3e0c8b0551896b9 size: 2007
[root@k8s-81 images]# 从hardor的gcdw项目里查看
可以看到所需要的所有镜像。
创建命名空间
kubectl create ns gcdw创建访问凭证
注意里面的命名空间，凭证的名字，harbor服务的地址，用户名和密码。
[root@k8s-81 ~]# kubectl create secret --namespace gcdw docker-registry gcdw-regcred --docker-server=172.16.3.249:8443 --docker-username=admin --docker-password=Admin12345
secret/gcdw-regcred created
[root@k8s-81 ~]#上传gcdw部署helm文件
[root@vm248 gcdw_9.8.0.3.5]# tree .
.
└── helm
└── single_tenant
├── bin
│ └── helm
├── chart
│ ├── Chart.yaml
│ ├── templates
│ │ ├── gcdw-operator
│ │ │ ├── controller_manager_config.yaml
│ │ │ ├── filebeat_configmap_gcware.yaml
│ │ │ ├── filebeat_configmap_warehouse.yaml
│ │ │ ├── filebeat_configmap.yaml
│ │ │ ├── gcdw_coordinator_configmap.yaml
│ │ │ ├── gcdw_warehouse_configmap.yaml
│ │ │ ├── init-config.yaml
│ │ │ ├── leader_election_role_binding.yaml
│ │ │ ├── leader_election_role.yaml
│ │ │ ├── manager.yaml
│ │ │ ├── role_binding.yaml
│ │ │ ├── role.yaml
│ │ │ └── sa.yaml
│ │ ├── gcdw-saas
│ │ │ ├── gcdw-fdb-cm.yaml
│ │ │ ├── gcdw-ftp-st.yaml
│ │ │ ├── gcdw-redis-cm.yaml
│ │ │ ├── gcdw-redis-st.yaml
│ │ │ ├── gcdw-saas-backend-cm.yaml
│ │ │ ├── gcdw-saas-backend-st.yaml
│ │ │ ├── gcdw-saas-frontend-cm.yaml
│ │ │ ├── gcdw-saas-frontend-st.yaml
│ │ │ └── rbac.yaml
│ │ └── _helpers.tpl
│ └── values.yaml
├── crds
│ ├── gcdw.gbase.cn_coordinators.yaml
│ └── gcdw.gbase.cn_warehouses.yaml
└── tools
└── s3_monit_cronjob.yaml
9 directories, 29 files
[root@vm248 gcdw_9.8.0.3.5]#
编辑char/values.yaml文件
修改内容如下
registry
镜像服务器的地址
imagePullSecrets
访问镜像服务器的安全凭证，用上一步创建凭证的那个名字。
fdb_addr
foundationdb的地址
fdb_cluster
/etc/foundationdb/fdb.cluster文件里的访问密钥串。
cpu
所需的CPU,包括最小和最大，我这里最小都改成了1，因为本机资源不足。 实际场景根据资源情况调整。
memory
内存限制，同CPU,我最小的都改成了1G
replicas
副本数量，我单机，就改成了1。
完整的values.yaml的样例
[root@k8s-81 opt]# cat /opt/gcdw/helm/single_tenant/chart/values.yaml
# Default values for gcdw
global:
cluster_domain: "cluster.local"
image:
registry: "172.16.3.249:8443"
pullPolicy: Always
imagePullSecrets: "gcdw-regcred"
metadata:
fdb_addr: "172.16.3.249:4550"
fdb_cluster: "BougYK7P:jFV4Ipbn@172.16.3.249:4500"
privatePlatform: true
# gcdw operator values-------------
operator:
enabled: true
image:
repository: gcdw/gcdw-operator
tag: 9.8.0.3.5
replicas: 1
leaderElect: true
resources:
requests:
cpu: 1
memory: 1Gi
limits:
cpu: 2
memory: 2Gi
# gcdw server values-------------------
server:
image:
repository: gcdw/gcdw-server
tag: 9.8.0.3.5
gcware_replicas: 1
coordinator_replicas: 1
gcware_resource:
request:
cpu: "1"
mem: "1Gi"
limit:
cpu: "2"
mem: 4Gi
coordinator_resource:
request:
cpu: "1"
mem: 1Gi
limit:
cpu: "4"
mem: 16Gi
warehouse_resource:
request:
cpu: "1"
mem: 1Gi
limit:
cpu: "16"
mem: 32Gi
# gcdw saas values--------------------
saas:
enabled: true
image:
frontend:
repository: gcdw/clound-database-frontend
tag: 9.8.0.3.5
backend:
repository: gcdw/clound-database-backend
tag: 9.8.0.3.5
redis:
repository: gcdw/redis
tag: latest
vsftpd:
repository: gcdw/vsftpd
tag: latest
replicas:
frontend: 1
backend: 1
redis: 1
vsftpd: 1
webServicePort: 32143
# logging system setting
logging:
enabled: false
filebeatImage: ""
filebeatOutput:
output.elasticsearch:
hosts:
- ""
username: ""
password: ""
index: 'gcdw-${MY_POD_NAMESPACE}-${MY_POD_NAME}-log-%{+yyyy.MM.dd}'
logrotateImage: "gcdw/gcdw-logrotate"
# persistant storage setting
storage:
enabled: false
storageClass: "rook-ceph-block"
storageSize: "1Gi"
# node affinity setting
nodeAffinitySchedule:
cloudServiceAffinity:
key: cloudService
value: gcdw
coordiantorServiceAffinity:
key: coordinatorService
value: gcdw
warehouseServiceAffinity:
key: warehouseService
value: gcdw
[root@k8s-81 opt]#在 kubernetes 上创建自定义资源
运行目录就在chart的目录下
[root@k8s-81 single_tenant]# kubectl apply -f crds
customresourcedefinition.apiextensions.k8s.io/coordinators.gcdw.gbase.cn created
customresourcedefinition.apiextensions.k8s.io/warehouses.gcdw.gbase.cn created
[root@k8s-81 single_tenant]# 使用 helm 安装 GCDW 实例
[root@k8s-81 single_tenant]# ./bin/helm install gcdw chart -n gcdw
NAME: gcdw
LAST DEPLOYED: Thu Feb 23 16:03:05 2023
NAMESPACE: gcdw
STATUS: deployed
REVISION: 1
TEST SUITE: None
[root@k8s-81 single_tenant查看安装状态
[root@k8s-81 single_tenant]# ./bin/helm ls -n gcdw
NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
gcdw gcdw 1 2023-05-30 17:12:42.354513312 +0800 CST deployed gcdw-single-tenancy-9.8.0.3.5
[root@k8s-81 single_tenant]#
查看在 gcdw 命名空间下已安装的 helm 项目
其中几个基础服务，只有gcdw-operator、gcdw-saas-backend和gcdw-saas-frontend会运行，其它的要在创建用户后，才会自动启动服务。
注意POD的状态，会先下载镜像，再启动服务，最后才是Running状态。
WEB访问端口为32143。
[root@vm248 gcdw_9.8.0.3.5]# kubectl get all -n gccdw
No resources found in gccdw namespace.
[root@vm248 gcdw_9.8.0.3.5]# kubectl get all -n gcdw
NAME READY STATUS RESTARTS AGE
pod/gcdw-ftp-0 1/1 Running 0 3h39m
pod/gcdw-operator-b66b7b698-xhc88 1/1 Running 0 3h39m
pod/gcdw-redis-0 1/1 Running 0 3h39m
pod/gcdw-saas-backend-0 1/1 Running 0 3h39m
pod/gcdw-saas-frontend-0 1/1 Running 3 (3h39m ago) 3h39m
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/gcdw-1 NodePort 10.106.10.223 <none> 5258:32045/TCP 3h35m
service/gcdw-ftp ClusterIP None <none> 21/TCP 3h39m
service/gcdw-redis ClusterIP 10.101.28.232 <none> 6378/TCP 3h39m
service/gcdw-saas-backend ClusterIP None <none> 8082/TCP 3h39m
service/gcdw-saas-frontend NodePort 10.105.208.82 <none> 80:32143/TCP 3h39m
service/gcware ClusterIP None <none> 5919/TCP 3h35m
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/gcdw-operator 1/1 1 1 3h39m
NAME DESIRED CURRENT READY AGE
replicaset.apps/gcdw-operator-b66b7b698 1 1 1 3h39m
NAME READY AGE
statefulset.apps/gcdw-1 1/1 3h35m
statefulset.apps/gcdw-ftp 1/1 3h39m
statefulset.apps/gcdw-redis 1/1 3h39m
statefulset.apps/gcdw-saas-backend 1/1 3h39m
statefulset.apps/gcdw-saas-frontend 1/1 3h39m
statefulset.apps/gcware 1/1 3h35m
[root@vm248 gcdw_9.8.0.3.5]#
检查POD运行状态
如果状态一直不能变成Running,可以用describe查看状态。 其它命令请参考kubectl手册。常见的包括log查看日志, exec等。
[root@vm248 gcdw_9.8.0.3.5]# kubectl describe pods gcdw-saas-frontend-0 -n gcdw
Name: gcdw-saas-frontend-0
Namespace: gcdw
Priority: 0
Service Account: default
Node: vm248/172.16.3.248
Start Time: Sun, 26 Feb 2023 21:56:32 -0800
Labels: app=gcdw-saas-frontend
controller-revision-hash=gcdw-saas-frontend-74c459fc55
statefulset.kubernetes.io/pod-name=gcdw-saas-frontend-0
Annotations: <none>
Status: Running
IP: 10.244.0.56
IPs:
IP: 10.244.0.56
Controlled By: StatefulSet/gcdw-saas-frontend
Containers:
gcdw-saas-frontend:
Container ID: containerd://a5cad831f609f1b6c32ae5d00201aec969a8bbc81c67ae5bc86ef686cc7ebe6f
Image: 172.16.3.249:8443/gcdw/clound-database-frontend:9.8.0.3.5
Image ID: 172.16.3.249:8443/gcdw/clound-database-frontend@sha256:19986c92474cd6dce5781f976b32364f1275064ee79e67cdea19900f24403f04
Port: 80/TCP
Host Port: 0/TCP
Command:
nginx
-g
daemon off;
State: Running
Started: Sun, 26 Feb 2023 21:57:17 -0800
Last State: Terminated
Reason: Error
Exit Code: 1
Started: Sun, 26 Feb 2023 21:56:51 -0800
Finished: Sun, 26 Feb 2023 21:56:51 -0800
Ready: True
Restart Count: 3
Environment: <none>
Mounts:
/etc/nginx/nginx.conf from frontend (rw,path="nginx.conf")
/var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-pfg64 (ro)
Conditions:
Type Status
Initialized True
Ready True
ContainersReady True
PodScheduled True
Volumes:
frontend:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: gcdw-saas-frontend
Optional: false
kube-api-access-pfg64:
Type: Projected (a volume that contains injected data from multiple sources)
TokenExpirationSeconds: 3607
ConfigMapName: kube-root-ca.crt
ConfigMapOptional: <nil>
DownwardAPI: true
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events: <none>
[root@vm248 gcdw_9.8.0.3.5]#
登录web终端
http://10.0.2.81:32143