k8s部署rabbitmq集群

骑猪看日落 2023-02-14 01:34 468阅读 0赞

阿里云创建NAS共享存储的StorageClass：

apiVersion: storage.k8s.io/v1
    kind: StorageClass
    metadata:
      name: alicloud-nas-subpath-public
    provisioner: nasplugin.csi.alibabacloud.com
    mountOptions:
    - nolock,tcp,noresvport
    - vers=3
    parameters:
      volumeAs: subpath
      server: "xxxxxx.cn-hangzhou.nas.aliyuncs.com:/"
    reclaimPolicy: Retain

--------------------

部署rabbitmq集群：

public-service-ns.yaml

apiVersion: v1
    kind: Namespace
    metadata:
      name: public-service

rabbitmq-configmap.yaml

apiVersion: v1
    kind: ConfigMap
    metadata:
      name: rmq-cluster-config
      namespace: public-service
      labels:
        addonmanager.kubernetes.io/mode: Reconcile
    data:
        enabled_plugins: | [rabbitmq_management,rabbitmq_peer_discovery_k8s].
        rabbitmq.conf: | loopback_users.guest = false
    
          ## Clustering
          cluster_formation.peer_discovery_backend = rabbit_peer_discovery_k8s
          cluster_formation.k8s.host = kubernetes.default.svc.cluster.local
          cluster_formation.k8s.address_type = hostname
          #################################################
          # public-service is rabbitmq-cluster's namespace#
          #################################################
          cluster_formation.k8s.hostname_suffix = .rmq-cluster.public-service.svc.cluster.local
          cluster_formation.node_cleanup.interval = 10
          cluster_formation.node_cleanup.only_log_warning = true
          cluster_partition_handling = autoheal
          ## queue master locator
          queue_master_locator=min-masters

rabbitmq-rbac.yaml

apiVersion: v1
    kind: ServiceAccount
    metadata:
      name: rmq-cluster
      namespace: public-service
      
    ---
    apiVersion: rbac.authorization.k8s.io/v1beta1
    kind: Role
    metadata:
      name: rmq-cluster
      namespace: public-service
    rules:
      - apiGroups:
          - ""
        resources:
          - endpoints
        verbs:
          - get
    
    ---
    apiVersion: rbac.authorization.k8s.io/v1beta1
    kind: RoleBinding
    metadata:
      name: rmq-cluster
      namespace: public-service
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: Role
      name: rmq-cluster
    subjects:
    - kind: ServiceAccount
      name: rmq-cluster
      namespace: public-service

rabbitmq-secret.yaml

apiVersion: v1
    kind: Secret
    metadata:
      name: rmq-cluster-secret
      namespace: public-service
    stringData:
      cookie: ERLANG_COOKIE
      username: admin
      password: admin123
    type: Opaque

rabbitmq-svc.yaml

apiVersion: v1
    kind: Service
    metadata:
      name: rmq-cluster
      namespace: public-service
      labels:
        app: rmq-cluster
    spec:
      selector:
        app: rmq-cluster
      clusterIP: 172.21.11.245              #指定clusterIP，方便使用
      ports:
      - name: http
        port: 15672
        protocol: TCP
        targetPort: 15672
      - name: amqp
        port: 5672
        protocol: TCP
        targetPort: 5672
      type: ClusterIP

rabbitmq-sts.yaml

apiVersion: apps/v1
    kind: StatefulSet
    metadata:
      name: rmq-cluster
      namespace: public-service
      labels:
        app: rmq-cluster
    spec:
      replicas: 3
      selector:
        matchLabels:
          app: rmq-cluster
      serviceName: rmq-cluster
      template:
        metadata:
          labels:
            app: rmq-cluster
        spec:
          serviceAccountName: rmq-cluster
          terminationGracePeriodSeconds: 30
          containers:
          - name: rabbitmq
            image: rabbitmq:3.7-management
            imagePullPolicy: IfNotPresent
            ports:
            - containerPort: 15672
              name: http
              protocol: TCP
            - containerPort: 5672
              name: amqp
              protocol: TCP
            command:
            - sh
            args:
            - -c
            - cp -v /etc/rabbitmq/rabbitmq.conf ${ RABBITMQ_CONFIG_FILE}; exec docker-entrypoint.sh
              rabbitmq-server
            env:
            - name: RABBITMQ_DEFAULT_USER
              valueFrom:
                secretKeyRef:
                  key: username
                  name: rmq-cluster-secret
            - name: RABBITMQ_DEFAULT_PASS
              valueFrom:
                secretKeyRef:
                  key: password
                  name: rmq-cluster-secret
            - name: RABBITMQ_ERLANG_COOKIE
              valueFrom:
                secretKeyRef:
                  key: cookie
                  name: rmq-cluster-secret
            - name: K8S_SERVICE_NAME
              value: rmq-cluster
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            - name: RABBITMQ_USE_LONGNAME
              value: "true"
            - name: RABBITMQ_NODENAME
              value: rabbit@$(POD_NAME).rmq-cluster.$(POD_NAMESPACE).svc.cluster.local
            - name: RABBITMQ_CONFIG_FILE
              value: /var/lib/rabbitmq/rabbitmq.conf
            livenessProbe:
              exec:
                command:
                - rabbitmqctl
                - status
              initialDelaySeconds: 30
              timeoutSeconds: 10
            readinessProbe:
              exec:
                command:
                - rabbitmqctl
                - status
              initialDelaySeconds: 10
              timeoutSeconds: 10
            volumeMounts:
            - name: config-volume
              mountPath: /etc/rabbitmq
              readOnly: false
            - name: rabbitmq-storage
              mountPath: /var/lib/rabbitmq
              readOnly: false
          volumes:
          - name: config-volume
            configMap:
              items:
              - key: rabbitmq.conf
                path: rabbitmq.conf
              - key: enabled_plugins
                path: enabled_plugins
              name: rmq-cluster-config
          - name: rabbitmq-storage
            persistentVolumeClaim:
              claimName: rabbitmq-cluster-storage
    
    ---
    apiVersion: v1
    kind: PersistentVolumeClaim
    metadata:
      name: rabbitmq-cluster-storage
      namespace: public-service
    spec:
      accessModes:
      - ReadWriteMany
      resources:
        requests:
          storage: 20Gi
      storageClassName: alicloud-nas-subpath-public

rabbitmq-ingress.yaml

apiVersion: extensions/v1beta1
    kind: Ingress
    metadata:
      name: rabbitmq
      namespace: public-service
    spec:
      rules:
        - host: rabbitmq.lzxlinux.com
          http:
            paths:
              - path: /
                backend:
                  serviceName: rmq-cluster
                  servicePort: 15672

部署完毕后，

kubectl get all -n public-service
    
    NAME                READY   STATUS    RESTARTS   AGE
    pod/rmq-cluster-0   1/1     Running   0          4h
    pod/rmq-cluster-1   1/1     Running   0          4h
    pod/rmq-cluster-2   1/1     Running   0          4h
    
    NAME                  TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)              AGE
    service/rmq-cluster   ClusterIP   172.21.11.245   <none>        15672/TCP,5672/TCP   4h
    
    NAME                           READY   AGE
    statefulset.apps/rmq-cluster   3/3     4h

添加hosts：`rabbitmq.lzxlinux.com`，使用初始账号密码`guest/guest`登录即可。

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz_size_16_color_FFFFFF_t_70]

--------------------

补充：

1.  如果使用的是 default 命名空间，就使用 sed 命令将yaml文件中 public-service 全局替换为 default：

sed -i 's/public-service/default/g' ./*
    
    kubectl apply -f .

1.  如果使用的是nfs持久化存储，对于nfs目录赋予755权限，然后其他节点安装nfs即可：

选择一个节点上做nfs共享，

yum install -y nfs-utils rpcbind
    
    mkdir -p /data/rabbitmq
    
    vim /etc/exports

/data/rabbitmq 192.168.30.0/24(rw,sync,no_root_squash)

chmod -R 755 /data/rabbitmq
    
    exportfs -arv
    
    systemctl enable rpcbind && systemctl start rpcbind
    
    systemctl enable nfs && systemctl start nfs

nfs部署完毕。对于需要使用nfs的node节点，都要安装nfs：

yum install -y nfs-utils

rabbit-pv.yaml

apiVersion: v1
    kind: PersistentVolume
    metadata:
      name: rabbitmq-pv
      labels:
        app: rmq-cluster
    spec:
      capacity:
        storage: 5Gi
      accessModes:
        - ReadWriteMany
      nfs:
        server: 192.168.30.129
        path: /data/rabbitmq

rabbitmq-pvc.yaml

apiVersion: v1
    kind: PersistentVolumeClaim
    metadata:
      name: rabbitmq-cluster-storage
      namespace: default
    spec:
      accessModes:
      - ReadWriteMany
      resources:
        requests:
          storage: 5Gi
      selector:
        matchLabels:
          app: rmq-cluster

自己再次部署了一遍，没有遇到任何报错：

kubectl get pv
    
    NAME          CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                              STORAGECLASS   REASON   AGE
    rabbitmq-pv   5Gi        RWX            Retain           Bound    default/rabbitmq-cluster-storage                           50m
    
    kubectl get pvc
    
    NAME                       STATUS   VOLUME        CAPACITY   ACCESS MODES   STORAGECLASS   AGE
    rabbitmq-cluster-storage   Bound    rabbitmq-pv   5Gi        RWX                           51m
    
    kubectl get pod
    
    NAME            READY   STATUS    RESTARTS   AGE
    rmq-cluster-0   1/1     Running   0          51m
    rmq-cluster-1   1/1     Running   0          50m
    rmq-cluster-2   1/1     Running   0          50m
    
    kubectl get svc
    
    NAME          TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)              AGE
    kubernetes    ClusterIP   10.96.0.1      <none>        443/TCP              86d
    rmq-cluster   ClusterIP   10.96.11.245   <none>        15672/TCP,5672/TCP   51m

--------------------

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/20200530130320837.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz,size_16,color_FFFFFF,t_70