Helm 创建高可用redis集群

数据 在多个Redis节点之间自动分片

sentinel特点：

它的主要功能有以下几点

不时地监控redis是否按照预期良好地运行;
如果发现某个redis节点运行出现状况，能够通知另外一个进程(例如它的客户端);
能够进行自动切换。当一个master节点不可用时，能够选举出master的多个slave(如果有超过一个slave的话)中的一个来作为新的master,其它的slave节点会将它所追随的master的地址改为被提升为master的slave的新地址。

sentinel配置文件详解

参考：https://segmentfault.com/a/1190000002680804

主节点down了，从节点选举机制如下：

　　https://blog.csdn.net/tr1912/article/details/81265007

安装redis集群

我们首先添加一下helm库，并且搜索到redis

$ helm repo add bitnami https://charts.bitnami.com/bitnami
$ helm search repo redis
NAME                                CHART VERSION    APP VERSION    DESCRIPTION
bitnami/redis                       14.6.6           6.2.4          Open source, advanced key-value store. It is of...
bitnami/redis-cluster               6.2.3            6.2.4          Open source, advanced key-value store. It is of...
stable/prometheus-redis-exporter    3.5.0            1.3.4          Prometheus exporter for Redis metrics
stable/redis                        10.5.7           5.0.7          DEPRECATED Open source, advanced key-value stor...
stable/redis-ha                     4.4.6            5.0.6          DEPRECATED - Highly available Kubernetes implem...

helm search repo redis
helm pull stable/redis-ha
tar zxvf redis-ha-*.tgz
cp redis-ha/values.yaml .

cat value.yaml

## Configure resource requests and limits
## ref: http://kubernetes.io/docs/user-guide/compute-resources/
##
image:
  repository: redis
  tag: 5.0.6-alpine
  pullPolicy: IfNotPresent

## Reference to one or more secrets to be used when pulling images
## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
## This imagePullSecrets is only for redis images
##
imagePullSecrets: []
# - name: "image-pull-secret"

## replicas number for each component
replicas: 3

## Kubernetes priorityClass name for the redis-ha-server pod
# priorityClassName: ""

## Custom labels for the redis pod
labels: {}

## Pods Service Account
## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
serviceAccount:
  ## Specifies whether a ServiceAccount should be created
  ##
  create: true
  ## The name of the ServiceAccount to use.
  ## If not set and create is true, a name is generated using the redis-ha.fullname template
  # name:

## Enables a HA Proxy for better LoadBalancing / Sentinel Master support. Automatically proxies to Redis master.
## Recommend for externally exposed Redis clusters.
## ref: https://cbonte.github.io/haproxy-dconv/1.9/intro.html
haproxy:
  enabled: false
  # Enable if you want a dedicated port in haproxy for redis-slaves
  readOnly:
    enabled: false
    port: 6380
  replicas: 3
  image:
    repository: haproxy
    tag: 2.0.4
    pullPolicy: IfNotPresent

  ## Reference to one or more secrets to be used when pulling images
  ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
  ##
  imagePullSecrets: []
  # - name: "image-pull-secret"

  annotations: {}
  resources: {}
  emptyDir: {}
  ## Enable sticky sessions to Redis nodes via HAProxy
  ## Very useful for long-living connections as in case of Sentry for example
  stickyBalancing: false
  ## Kubernetes priorityClass name for the haproxy pod
  # priorityClassName: ""
  ## Service type for HAProxy
  ##
  service:
    type: ClusterIP
    loadBalancerIP:
    annotations: {}
  serviceAccount:
    create: true
  ## Official HAProxy embedded prometheus metrics settings.
  ## Ref: https://github.com/haproxy/haproxy/tree/master/contrib/prometheus-exporter
  ##
  metrics:
    enabled: false
    # prometheus port & scrape path
    port: 9101
    portName: exporter-port
    scrapePath: /metrics

    serviceMonitor:
      # When set true then use a ServiceMonitor to configure scraping
      enabled: false
      # Set the namespace the ServiceMonitor should be deployed
      # namespace: monitoring
      # Set how frequently Prometheus should scrape
      # interval: 30s
      # Set path to redis-exporter telemtery-path
      # telemetryPath: /metrics
      # Set labels for the ServiceMonitor, use this to define your scrape label for Prometheus Operator
      # labels: {}
      # Set timeout for scrape
      # timeout: 10s
  init:
    resources: {}
  timeout:
    connect: 4s
    server: 30s
    client: 30s
    check: 2s
  securityContext:
    runAsUser: 1000
    fsGroup: 1000
    runAsNonRoot: true

  ## Whether the haproxy pods should be forced to run on separate nodes.
  hardAntiAffinity: true

  ## Additional affinities to add to the haproxy pods.
  additionalAffinities: {}

  ## Override all other affinity settings for the haproxy pods with a string.
  affinity: |

  ## Custom config-haproxy.cfg files used to override default settings. If this file is
  ## specified then the config-haproxy.cfg above will be ignored.
  # customConfig: |-
      # Define configuration here
  ## Place any additional configuration section to add to the default config-haproxy.cfg
  # extraConfig: |-
      # Define configuration here


## Role Based Access
## Ref: https://kubernetes.io/docs/admin/authorization/rbac/
##
rbac:
  create: true

sysctlImage:
  enabled: false
  command: []
  registry: docker.io
  repository: busybox
  tag: 1.31.1
  pullPolicy: Always
  mountHostSys: false
  resources: {}

## Use an alternate scheduler, e.g. "stork".
## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/
##
# schedulerName:

## Redis specific configuration options
redis:
  port: 6379
  masterGroupName: "mymaster"       # must match ^[\\w-\\.]+$) and can be templated
  config:
    ## Additional redis conf options can be added below
    ## For all available options see http://download.redis.io/redis-stable/redis.conf
    min-replicas-to-write: 1
    min-replicas-max-lag: 5   # Value in seconds
    maxmemory: "0"       # Max memory to use for each redis instance. Default is unlimited.
    maxmemory-policy: "volatile-lru"  # Max memory policy to use for each redis instance. Default is volatile-lru.
    # Determines if scheduled RDB backups are created. Default is false.
    # Please note that local (on-disk) RDBs will still be created when re-syncing with a new slave. The only way to prevent this is to enable diskless replication.
    save: "900 1"
    # When enabled, directly sends the RDB over the wire to slaves, without using the disk as intermediate storage. Default is false.
    repl-diskless-sync: "yes"
    rdbcompression: "yes"
    rdbchecksum: "yes"


  ## Custom redis.conf files used to override default settings. If this file is
  ## specified then the redis.config above will be ignored.
  # customConfig: |-
      # Define configuration here

  resources: {}
  #  requests:
  #    memory: 200Mi
  #    cpu: 100m
  #  limits:
  #    memory: 700Mi

## Sentinel specific configuration options
sentinel:
  port: 26379
  quorum: 2
  config:
    ## Additional sentinel conf options can be added below. Only options that
    ## are expressed in the format simialar to 'sentinel xxx mymaster xxx' will
    ## be properly templated expect maxclients option.
    ## For available options see http://download.redis.io/redis-stable/sentinel.conf
    down-after-milliseconds: 10000
    ## Failover timeout value in milliseconds
    failover-timeout: 180000
    parallel-syncs: 5
    maxclients: 10000

  ## Custom sentinel.conf files used to override default settings. If this file is
  ## specified then the sentinel.config above will be ignored.
  # customConfig: |-
      # Define configuration here

  resources: {}
  #  requests:
  #    memory: 200Mi
  #    cpu: 100m
  #  limits:
  #    memory: 200Mi

securityContext:
  runAsUser: 1000
  fsGroup: 1000
  runAsNonRoot: true

## Node labels, affinity, and tolerations for pod assignment
## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#taints-and-tolerations-beta-feature
## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
nodeSelector: {}

## Whether the Redis server pods should be forced to run on separate nodes.
## This is accomplished by setting their AntiAffinity with requiredDuringSchedulingIgnoredDuringExecution as opposed to preferred.
## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#inter-pod-affinity-and-anti-affinity-beta-feature
##
hardAntiAffinity: true

## Additional affinities to add to the Redis server pods.
##
## Example:
##   nodeAffinity:
##     preferredDuringSchedulingIgnoredDuringExecution:
##       - weight: 50
##         preference:
##           matchExpressions:
##             - key: spot
##               operator: NotIn
##               values:
##                 - "true"
##
## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
##
additionalAffinities: {}

## Override all other affinity settings for the Redis server pods with a string.
##
## Example:
## affinity: |
##   podAntiAffinity:
##     requiredDuringSchedulingIgnoredDuringExecution:
##       - labelSelector:
##           matchLabels:
##             app: {{ template "redis-ha.name" . }}
##             release: {{ .Release.Name }}
##         topologyKey: kubernetes.io/hostname
##     preferredDuringSchedulingIgnoredDuringExecution:
##       - weight: 100
##         podAffinityTerm:
##           labelSelector:
##             matchLabels:
##               app:  {{ template "redis-ha.name" . }}
##               release: {{ .Release.Name }}
##           topologyKey: failure-domain.beta.kubernetes.io/zone
##
affinity: |

# Prometheus exporter specific configuration options
exporter:
  enabled: false
  image: oliver006/redis_exporter
  tag: v1.3.2
  pullPolicy: IfNotPresent

  # prometheus port & scrape path
  port: 9121
  scrapePath: /metrics

  # cpu/memory resource limits/requests
  resources: {}

  # Additional args for redis exporter
  extraArgs: {}

  # Used to mount a LUA-Script via config map and use it for metrics-collection
  # script: |
  #   -- Example script copied from: https://github.com/oliver006/redis_exporter/blob/master/contrib/sample_collect_script.lua
  #   -- Example collect script for -script option
  #   -- This returns a Lua table with alternating keys and values.
  #   -- Both keys and values must be strings, similar to a HGETALL result.
  #   -- More info about Redis Lua scripting: https://redis.io/commands/eval
  #
  #   local result = {}
  #
  #   -- Add all keys and values from some hash in db 5
  #   redis.call("SELECT", 5)
  #   local r = redis.call("HGETALL", "some-hash-with-stats")
  #   if r ~= nil then
  #   for _,v in ipairs(r) do
  #   table.insert(result, v) -- alternating keys and values
  #   end
  #   end
  #
  #   -- Set foo to 42
  #   table.insert(result, "foo")
  #   table.insert(result, "42") -- note the string, use tostring() if needed
  #
  #   return result

  serviceMonitor:
    # When set true then use a ServiceMonitor to configure scraping
    enabled: false
    # Set the namespace the ServiceMonitor should be deployed
    # namespace: monitoring
    # Set how frequently Prometheus should scrape
    # interval: 30s
    # Set path to redis-exporter telemtery-path
    # telemetryPath: /metrics
    # Set labels for the ServiceMonitor, use this to define your scrape label for Prometheus Operator
    # labels: {}
    # Set timeout for scrape
    # timeout: 10s

podDisruptionBudget: {}
  # maxUnavailable: 1
  # minAvailable: 1

## Configures redis with AUTH (requirepass & masterauth conf params)
auth: false
# redisPassword:

## Use existing secret containing key `authKey` (ignores redisPassword)
# existingSecret:

## Defines the key holding the redis password in existing secret.
authKey: auth

persistentVolume:
  enabled: true
  ## redis-ha data Persistent Volume Storage Class
  ## If defined, storageClassName: <storageClass>
  ## If set to "-", storageClassName: "", which disables dynamic provisioning
  ## If undefined (the default) or set to null, no storageClassName spec is
  ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
  ##   GKE, AWS & OpenStack)
  ##
  # storageClass: "-"
  accessModes:
    - ReadWriteOnce
  size: 10Gi
  annotations: {}
  # reclaimPolicy per https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming
  reclaimPolicy: ""
init:
  resources: {}

# To use a hostPath for data, set persistentVolume.enabled to false
# and define hostPath.path.
# Warning: this might overwrite existing folders on the host system!
hostPath:
  ## path is evaluated as template so placeholders are replaced
  # path: "/data/{{ .Release.Name }}"

  # if chown is true, an init-container with root permissions is launched to
  # change the owner of the hostPath folder to the user defined in the
  # security context
  chown: true

emptyDir: {}

1. 修改 “hardAntiAffinity: true” 为 “hardAntiAffinity: false” (仅限当replicas > worker node 节点数时修改)
2. 修改 “auth: false” 为 “auth: true”，打开 “# redisPassword:” 的注释并设置密码
1. 打开 “ # storageClass: “-“ ” 的注释，并修改 “-” 为 集群中的自动供给卷 “managed-nfs-storage”， 配置中 “size: 10Gi” 的大小为默认设置，可根据需要进行调整

接着，我们来通过如下helm命令来创建redis集群，

helm install --namespace  redis redis-ha -f ./values.yaml ./redis-ha
helm upgrade redis-ha --namespace redis -f values.yaml ./redis-ha
helm delete redis-ha --namespace redis

创建成功之后，会有如下输出，

NAME: test-redis
LAST DEPLOYED: Sat Jul 17 21:56:12 2021
NAMESPACE: redis
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
** Please be patient while the chart is being deployed **

Redis(TM) can be accessed on the following DNS names from within your cluster:

    test-redis-master.redis.svc.cluster.local for read/write operations (port 6379)
    test-redis-replicas.redis.svc.cluster.local for read-only operations (port 6379)



To get your password run:

    export REDIS_PASSWORD=$(kubectl get secret --namespace redis test-redis -o jsonpath="{.data.redis-password}" | base64 --decode)

To connect to your Redis(TM) server:

1. Run a Redis(TM) pod that you can use as a client:

   kubectl run --namespace redis redis-client --restart='Never'  --env REDIS_PASSWORD=$REDIS_PASSWORD  --image docker.io/bitnami/redis:6.2.4-debian-10-r13 --command -- sleep infinity

   Use the following command to attach to the pod:

   kubectl exec --tty -i redis-client \
   --namespace redis -- bash

2. Connect using the Redis(TM) CLI:
   redis-cli -h test-redis-master -a $REDIS_PASSWORD
   redis-cli -h test-redis-replicas -a $REDIS_PASSWORD

To connect to your database from outside the cluster execute the following commands:

    export NODE_IP=$(kubectl get nodes --namespace redis -o jsonpath="{.items[0].status.addresses[0].address}")
    export NODE_PORT=$(kubectl get --namespace redis -o jsonpath="{.spec.ports[0].nodePort}" services test-redis-master)
    redis-cli -h $NODE_IP -p $NODE_PORT -a $REDIS_PASSWORD

这样等待redis集群的创建了，我们也可以通过命令查看是否创建成功：

$ kubectl get all -n redis
NAME                        READY   STATUS    RESTARTS   AGE
pod/test-redis-master-0     1/1     Running   0          24h
pod/test-redis-replicas-0   1/1     Running   1          24h
pod/test-redis-replicas-1   1/1     Running   0          24h
pod/test-redis-replicas-2   1/1     Running   0          23h

NAME                          TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE
service/test-redis-headless   ClusterIP   None           <none>        6379/TCP         24h
service/test-redis-master     NodePort    172.21.0.201   <none>        6379:30676/TCP   24h
service/test-redis-replicas   NodePort    172.21.6.241   <none>        6379:32112/TCP   24h

NAME                                   READY   AGE
statefulset.apps/test-redis-master     1/1     24h
statefulset.apps/test-redis-replicas   3/3     24h

这样创建全部成功，接下来就是使用redis了。

验证 redis-ha

#查看所有 pod
kubectl get pods
#执行结果
NAME                                      READY   STATUS    RESTARTS   AGE
nfs-client-provisioner-779bcc9dbb-vfjtl   1/1     Running   3          2d20h
redis-ha-server-0                         3/3     Running   1          5h32m
redis-ha-server-1                         3/3     Running   0          5h32m

#进入 redis-ha-server-0 容器内
 kubectl exec -it redis-ha-server-0  sh
 执行结果
 Defaulting container name to redis.
Use 'kubectl describe pod/redis-ha-server-0 -n default' to see all of the containers in this pod.
/data $ 

# redis-cli 测试
/data $ redis-cli
127.0.0.1:6379> auth xxxxxxxxxxx(此处为 values.yaml 文件中设置过的密码)
OK
127.0.0.1:6379> keys *
1) "test"
127.0.0.1:6379> get test
"111"
127.0.0.1:6379> set test 222
OK
127.0.0.1:6379> get test
"222"
127.0.0.1:6379>

如果需要暴露给外部使用则需要再部署一个 NodePort Service

[root@k8s-master redis-cluster]# cat service.yaml 
apiVersion: v1
kind: Service
metadata:
  name: redis-ha-service    #名称：随意
  namespace: redis   #名称：随意
  labels:
    app: redis-ha           #部署的 redis-ha 名称
spec:
  ports:
  - name: redis-ha          #部署的 redis-ha 名称
    protocol: "TCP"           #TCP 协议
    port: 26379             
    targetPort: 6379        
    nodePort: 30379         #此为外部连接k8s redis-ha 服务的端口
  selector:
    statefulset.kubernetes.io/pod-name: redis-ha-server-0
  type: NodePort

#部署 Service
kubectl apply -f service.yaml

#查看部署结果
kubectl get svc
执行结果
NAME                  TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                       AGE
kubernetes            ClusterIP   10.96.0.1        <none>        443/TCP                       2d21h
redis-ha              ClusterIP   None             <none>        6379/TCP,26379/TCP,9121/TCP   5h47m
redis-ha-announce-0   ClusterIP   10.99.18.87      <none>        6379/TCP,26379/TCP,9121/TCP   5h47m
redis-ha-announce-1   ClusterIP   10.100.130.186   <none>        6379/TCP,26379/TCP,9121/TCP   5h47m
redis-ha-service      NodePort    10.109.28.12     <none>        26379:30379/TCP               3h25m

#此时可以通过 k8s 任意 master 节点 IP:30379 端口进行连接

故障转移实验

停止主redis：

#1、在主上故障转移测试，使主down掉
redis-cli -h redis-ha-announce-0 -p 6379 debug segfault

#2、然后进入redis容器或sentinel容器
kubectl exec -it redis-ha-server-2 -c redis sh
kubectl exec -it redis-ha-server-0 -c sentinel sh

#3、容器里面使用redis客户端连接redis服务端
redis-cli -h redis-ha-announce-1 -p 6379 
redis-cli -h redis-ha-announce-2 -p 6379
#容器里面使用redis客户端连接Sentinel服务端
redis-cli -h redis-ha-announce-0 -p 26379


#4、redis查看主从状态
info replication
#sentinel查看状态 
INFO Sentinel

#5、查看日志
kubectl logs -f  redis-ha-server-0 -c redis
kubectl logs -f  redis-ha-server-1 -c redis
kubectl logs -f  redis-ha-server-2 -c redis

#主前任主上查看redis主从状态如下，10.106.29.241是第三台redis的ip，说转换成功了