k8s 1.28 搭建rabbitmq集群

1.环境

1.1 k8s 1.28

1.2 rabbit 3.8

1.3 工作空间default

1.4 注意，内存最好充足一点，因为我就两个节点一个master、一个node，起初我的node是8g，还剩3~4G，集群竟然一直起不来，后来将虚拟机内存扩大，并重启，rabbit集群就起来了，当然可能是和重启有关，但是我认为还是因为内存不够了。

1.5 k8s集成nfs存储，并且sc的名称 managed-nfs-storage。

1.6 镜像 docker pull registry.cn-beijing.aliyuncs.com/dotbalo/rabbitmq:3.8

2.准备yaml

2.1 01-rabbitmq-configmap.yaml

注：图中这个cluster_formation.k8s.host 我一开始用的是kubernetes.default.svc.cluster.local，然后就是各种连不上，后来换上 kubernetes.default.svc就可以了，不知道是不是k8s新版本的问题。

kind: ConfigMap
apiVersion: v1
metadata:name: rabbitmq-cluster-confignamespace: defaultlabels:addonmanager.kubernetes.io/mode: Reconcile
data:enabled_plugins: |[rabbitmq_management,rabbitmq_peer_discovery_k8s].rabbitmq.conf: |default_user = admindefault_pass = 123!@### Cluster formation. See https://www.rabbitmq.com/cluster-formation.html to learn more.cluster_formation.peer_discovery_backend = rabbit_peer_discovery_k8s#cluster_formation.k8s.host = kubernetes.default.svc.cluster.localcluster_formation.k8s.host = kubernetes.default.svc## Should RabbitMQ node name be computed from the pod's hostname or IP address?## IP addresses are not stable, so using [stable] hostnames is recommended when possible.## Set to "hostname" to use pod hostnames.## When this value is changed, so should the variable used to set the RABBITMQ_NODENAME## environment variable.cluster_formation.k8s.address_type = hostname## How often should node cleanup checks run?cluster_formation.node_cleanup.interval = 30## Set to false if automatic removal of unknown/absent nodes## is desired. This can be dangerous, see##  * https://www.rabbitmq.com/cluster-formation.html#node-health-checks-and-cleanup##  * https://groups.google.com/forum/#!msg/rabbitmq-users/wuOfzEywHXo/k8z_HWIkBgAJcluster_formation.node_cleanup.only_log_warning = truecluster_partition_handling = autoheal## See https://www.rabbitmq.com/ha.html#master-migration-data-localityqueue_master_locator=min-masters## See https://www.rabbitmq.com/access-control.html#loopback-usersloopback_users.guest = falsecluster_formation.randomized_startup_delay_range.min = 0cluster_formation.randomized_startup_delay_range.max = 2# default is rabbitmq-cluster's namespace# hostname_suffixcluster_formation.k8s.hostname_suffix = .rabbitmq-cluster.default.svc.cluster.local# memoryvm_memory_high_watermark.absolute = 100MB# diskdisk_free_limit.absolute = 2GB

2.2 02-rabbitmq-service.yaml

kind: Service
apiVersion: v1
metadata:labels:app: rabbitmq-clustername: rabbitmq-clusternamespace: default
spec:clusterIP: Noneports:- name: rmqportport: 5672targetPort: 5672selector:app: rabbitmq-cluster---
kind: Service
apiVersion: v1
metadata:labels:app: rabbitmq-clustername: rabbitmq-cluster-managenamespace: default
spec:ports:- name: httpport: 15672protocol: TCPtargetPort: 15672selector:app: rabbitmq-clustertype: NodePort

2.3 03-rabbitmq-rbac.yaml

kind: Service
apiVersion: v1
metadata:labels:app: rabbitmq-clustername: rabbitmq-clusternamespace: default
spec:clusterIP: Noneports:- name: rmqportport: 5672targetPort: 5672selector:app: rabbitmq-cluster---
kind: Service
apiVersion: v1
metadata:labels:app: rabbitmq-clustername: rabbitmq-cluster-managenamespace: default
spec:ports:- name: httpport: 15672protocol: TCPtargetPort: 15672selector:app: rabbitmq-clustertype: NodePort
[root@master rabbitmq]# cat 03-rabbitmq-rbac.yaml 
apiVersion: v1
kind: ServiceAccount
metadata:name: rabbitmq-clusternamespace: default
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: rabbitmq-clusternamespace: default
rules:
- apiGroups: [""]resources: ["endpoints"]verbs: ["get"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: rabbitmq-clusternamespace: default
roleRef:apiGroup: rbac.authorization.k8s.iokind: Rolename: rabbitmq-cluster
subjects:
- kind: ServiceAccountname: rabbitmq-clusternamespace: default

2.4 04-rabbitmq-cluster-sts.yaml

apiVersion: v1
kind: ServiceAccount
metadata:name: rabbitmq-clusternamespace: default
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: rabbitmq-clusternamespace: default
rules:
- apiGroups: [""]resources: ["endpoints"]verbs: ["get"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: rabbitmq-clusternamespace: default
roleRef:apiGroup: rbac.authorization.k8s.iokind: Rolename: rabbitmq-cluster
subjects:
- kind: ServiceAccountname: rabbitmq-clusternamespace: default
[root@master rabbitmq]# cat 04-rabbitmq-cluster-sts.yaml
kind: StatefulSet
apiVersion: apps/v1
metadata:labels:app: rabbitmq-clustername: rabbitmq-clusternamespace: default
spec:replicas: 3selector:matchLabels:app: rabbitmq-clusterserviceName: rabbitmq-clustertemplate:metadata:labels:app: rabbitmq-clusterspec:containers:- args:- -c- cp -v /etc/rabbitmq/rabbitmq.conf ${RABBITMQ_CONFIG_FILE}; exec docker-entrypoint.shrabbitmq-servercommand:- shenv:- name: TZvalue: 'Asia/Shanghai'- name: RABBITMQ_ERLANG_COOKIEvalue: 'SWvCP0Hrqv43NG7GybHC95ntCJKoW8UyNFWnBEWG8TY='- name: K8S_SERVICE_NAMEvalue: rabbitmq-cluster- name: POD_IPvalueFrom:fieldRef:fieldPath: status.podIP- name: POD_NAMEvalueFrom:fieldRef:fieldPath: metadata.name- name: POD_NAMESPACEvalueFrom:fieldRef:fieldPath: metadata.namespace- name: RABBITMQ_USE_LONGNAMEvalue: "true"- name: RABBITMQ_NODENAMEvalue: rabbit@$(POD_NAME).$(K8S_SERVICE_NAME).$(POD_NAMESPACE).svc.cluster.local- name: RABBITMQ_CONFIG_FILEvalue: /var/lib/rabbitmq/rabbitmq.confimage: 192.168.2.73:80/library/rabbitmq:3.8imagePullPolicy: IfNotPresentlivenessProbe:exec:command:- rabbitmq-diagnostics- status# See https://www.rabbitmq.com/monitoring.html for monitoring frequency recommendations.initialDelaySeconds: 60periodSeconds: 60timeoutSeconds: 15name: rabbitmqports:- containerPort: 15672name: httpprotocol: TCP- containerPort: 5672name: amqpprotocol: TCPreadinessProbe:exec:command:- rabbitmq-diagnostics- statusinitialDelaySeconds: 20periodSeconds: 60timeoutSeconds: 10volumeMounts:- mountPath: /etc/rabbitmqname: config-volumereadOnly: false- mountPath: /var/lib/rabbitmqname: rabbitmq-storagereadOnly: false- name: timezonemountPath: /etc/localtimereadOnly: trueserviceAccountName: rabbitmq-clusterterminationGracePeriodSeconds: 30volumes:- name: config-volumeconfigMap:items:- key: rabbitmq.confpath: rabbitmq.conf- key: enabled_pluginspath: enabled_pluginsname: rabbitmq-cluster-config- name: timezonehostPath:path: /usr/share/zoneinfo/Asia/ShanghaivolumeClaimTemplates:- metadata:name: rabbitmq-storagespec:accessModes:- ReadWriteManystorageClassName: "managed-nfs-storage"resources:requests:storage: 2Gi