构建 ElastiCache for Redis Cluster 代理服务集群（下篇）

# 切换到root用户
~ sudo su 

# 修改open file和max user process
~ vim /etc/security/limits.conf
root soft nproc 65000
root hard nproc 65000
root soft nofile 65000
root hard nofile 65000

# 安装 golang 最新的 SDK
~ add-apt-repository ppa:longsleep/golang-backports
~ apt-get update
~ apt-get install golang # 1.17.1
~ cd ~ && mkdir -p go/src/bilibili/ && cd /root/go/src/bilibili/
~ git clone  https://github.com/bilibili/overlord.git

# 需要 checkout 最新指定版本
~ cd overlord && git checkout v1.9.4 && git status
~ cd cmd/proxy/
~ go mod download bou.ke/monkey
~ go get overlord/proxy
~ go build

# 创建统一部署目录 /opt/systemd_sh，部署编译后的 proxy 执行文件
mkdir /opt/systemd_sh/
cp proxy /opt/systemd_sh/proxy

# Overlord 的 Proxy 相关配置
~ vim /opt/systemd_sh/conf_proxy.toml
pprof = "0.0.0.0:2110"
# 服务状态监听端口，默认Prometheus监听端口
stat = "0.0.0.0:8080"
stdout = true
debug = true
log = "/opt/systemd_sh/overlord.log"
log_lv = 0

[proxy]
# The read timeout value in msec that we wait for to receive a response from the client. By default, we wait indefinitely.
read_timeout = 0
# The write timeout value in msec that we wait for to write a response to the client. By default, we wait indefinitely.
write_timeout = 0
# proxy accept max connections from client. By default, we no limit.
max_connections = 0
# proxy support prometheus metrics. By default, we use it.
use_metrics = true

# Overlord 的 Cluster 相关配置
~ vim /opt/systemd_sh/conf_cluster.toml
[[clusters]]
# This be used to specify the name of cache cluster.
name = "test-redis-cluster"
# The name of the hash function. Possible values are: sha1.
hash_method = "fnv1a_64"
# The key distribution mode. Possible values are: ketama.
hash_distribution = "ketama"
# A two character string that specifies the part of the key used for hashing. Eg "{}".
hash_tag = "{}"
# cache type: memcache | memcache_binary | redis | redis_cluster
cache_type = "redis_cluster"
# proxy listen proto: tcp | unix
listen_proto = "tcp"
# proxy listen addr: tcp addr | unix sock path
listen_addr = "0.0.0.0:6379"
# Authenticate to the Redis server on connect.
redis_auth = ""
# The dial timeout value in msec that we wait for to establish a connection to the server. By default, we wait indefinitely.
dial_timeout = 1000
# The read timeout value in msec that we wait for to receive a response from a server. By default, we wait indefinitely.
read_timeout = 1000
# The write timeout value in msec that we wait for to write a response to a server. By default, we wait indefinitely.
write_timeout = 1000
# The number of connections that can be opened to each server. By default, we open at most 1 server connection.
node_connections = 2
# The number of consecutive failures on a server that would lead to it being temporarily ejected when auto_eject is set to true. Defaults to 3.
ping_fail_limit = 3
# A boolean value that controls if server should be ejected temporarily when it fails consecutively ping_fail_limit times.
ping_auto_eject = false

slowlog_slower_than = 10
# A list of server address, port (name:port or ip:port) for this server pool when cache type is redis_cluster.
servers = [
    # Redis Cluster 的 Configuration Endpoint
    "test-redis-cluster.w3cdfd.clustercfg.usw2.cache.amazonaws.com:6379"
]

# 启动命令
~ /opt/systemd_sh/proxy -cluster /opt/systemd_sh/conf_cluster.toml -conf /opt/systemd_sh/conf_proxy.toml

# 测试服务可用
~ redis-cli -h localhost
localhost:6379> MSET a a-value b b-value c c-value
OK
localhost:6379> MGET a b c
1) "a-value"
2) "b-value"
3) "c-value"

# 创建启动脚本
~ cd /opt/systemd_sh/
~ vim overlord_proxy_start.sh
#!/bin/bash
/opt/systemd_sh/overlord_proxy_stop.sh
/opt/systemd_sh/proxy -cluster /opt/systemd_sh/conf_cluster.toml -conf /opt/systemd_sh/conf_proxy.toml

~ chmod +x overlord_proxy_start.sh

# systemctl 配置文件
~ vim /lib/systemd/system/overlord-proxy.service
[Unit]
Description=overlord-proxy service

[Service]
Type=simple
ExecStart=/opt/systemd_sh/overlord_proxy_start.sh
Restart=always
RestartSec=5
StartLimitInterval=30
PrivateTmp=true

[Install]
WantedBy=multi-user.target
Alias=overlord-proxy.service

# 添加到启动列表，启动服务
systemctl enable overlord-proxy
systemctl start overlord-proxy

# 下载 cloudwatch agent 安装包，本地安装
~ wget https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb
~ dpkg -i -E ./amazon-cloudwatch-agent.deb
~ cd /opt/aws/amazon-cloudwatch-agent/etc/

# 启动 cloudwatch agent 服务
~ /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s
~ systemctl status amazon-cloudwatch-agent

注意安装后系统目录的文件结构

├── etc
│  ├── amazon-cloudwatch-agent.d
│  │  └── amazon-cloudwatch-agent.json
│  ├── amazon-cloudwatch-agent.toml
│  ├── common-config.toml
│  └── env-config.json

# 更新上报参数的json配置文件
~ vim amazon-cloudwatch-agent.d/amazon-cloudwatch-agent.json
{
    "agent": {
        "metrics_collection_interval": 30,
        "run_as_user": "cwagent"
    },
    "metrics": {
        "append_dimensions": {
            "AutoScalingGroupName": "${aws:AutoScalingGroupName}",
            "ImageId": "${aws:ImageId}",
            "InstanceId": "${aws:InstanceId}",
            "InstanceType": "${aws:InstanceType}"
        },
        "aggregation_dimensions" : [
                    ["AutoScalingGroupName"],
                    ["AutoScalingGroupName", "InstanceType"],
                    ["InstanceId"],
                    ["InstanceType"],
                    ["InstanceId","InstanceType"]
            ],
        "metrics_collected": {
            "cpu": {
                "measurement": [
                    "cpu_usage_idle",
                    "cpu_usage_iowait",
                    "cpu_usage_user",
                    "cpu_usage_system"
                ],
                "metrics_collection_interval": 30,
                "resources": [
                    "*"
                ],
                "totalcpu": true
            },
            "mem": {
                "measurement": [
                    "mem_used_percent",
                    "mem_used",
                    "mem_cached",
                    "mem_total"
                ],
                "metrics_collection_interval": 5,
                "resources": [
                    "*"
                ]
            },
            "netstat": {
                "measurement": [
                    "tcp_established",
                    "tcp_time_wait"
                ],
                "metrics_collection_interval": 30
            }
        }
    }
}

# 修改配置文件权限
~ chown cwagent:cwagent amazon-cloudwatch-agent.d/amazon-cloudwatch-agent.json
~ rm amazon-cloudwatch-agent.d/default

# 重启服务，确认新配置生效
sudo systemctl restart amazon-cloudwatch-agent
sudo systemctl status amazon-cloudwatch-agent

# 修改 overlord proxy 和 cloudwatch agent 服务的 systemctl 配置
~ systemctl stop overlord-proxy
~ systemctl disable overlord-proxy

~ systemctl stop amazon-cloudwatch-agent
~ systemctl disable amazon-cloudwatch-agent
# 修改 proxy.conf 文件，添加占位符，后续用 CloudFormation 模板进行自动替换
~ vim /opt/systemd_sh/conf_cluster.toml
node_connections = NODE-CONN-COUNT
servers = [
    "REDIS-ENDPOINT"
]

~ vim overlord-cluster-cloudformation.yaml
AWSTemplateFormatVersion: 2010-09-09
Resources:
  RedisProxyAutoScalingGroupConfiguration:
    Type: 'AWS::AutoScaling::LaunchConfiguration'
    Properties:
      KeyName: !Ref RedisProxyKey
      ImageId: ami-0eced476e4e29e6342   # 创建的 Overlord Proxy 基础镜像AMI ID
      IamInstanceProfile: !Ref OverlordInstanceProfile
      SecurityGroups:
        - !Ref RedisProxySecurityGroup
      InstanceType: !Ref RedisClusterProxyInstanceType
      UserData:
        Fn::Base64: 
          !Sub  |
            #!/bin/bash -xe
            sed -i 's/REDIS-ENDPOINT/${RedisClusterConfigurationEndPoint}/g' /opt/systemd_sh/conf_cluster.toml
            sed -i 's/NODE-CONN-COUNT/${RedisClusterProxyNodeConnCount}/g' /opt/systemd_sh/conf_cluster.toml
            systemctl enable overlord-proxy
            systemctl start overlord-proxy
            systemctl enable amazon-cloudwatch-agent
            systemctl start amazon-cloudwatch-agent
            
  OverlordInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: "/"
      Roles: 
        - CloudWatchAgentServerRole # EC2 Role 权限名称

  RedisProxyAutoScalingPolicyCPUUtilization:
    Type: AWS::AutoScaling::ScalingPolicy
    Properties:
      AutoScalingGroupName: !Ref RedisProxyAutoScalingGroup
      PolicyType: TargetTrackingScaling
      TargetTrackingConfiguration:
        PredefinedMetricSpecification:
          PredefinedMetricType: ASGAverageCPUUtilization
        TargetValue: !Ref RedisClusterProxyAutoScallingCPUThreshold

  RedisProxyTargetGroup:
    Type: 'AWS::ElasticLoadBalancingV2::TargetGroup'
    Properties:
      HealthCheckEnabled: true
      HealthCheckIntervalSeconds: 10
      VpcId: !Ref RedisProxyVPC
      Protocol: TCP
      Port: 6379

  RedisProxyNLB:
    Type: 'AWS::ElasticLoadBalancingV2::LoadBalancer'
    Properties:
      Name: !Ref RedisProxyClusterName
      Type: network
      Scheme: internal
      Subnets: !Split [',', !Join [',', !Ref RedisProxySubnets]]

  RedisProxyELBListener:
    Type: 'AWS::ElasticLoadBalancingV2::Listener'
    DependsOn:
      - RedisProxyNLB
      - RedisProxyTargetGroup
    Properties:
      DefaultActions:
        - Type: forward
          TargetGroupArn: !Ref RedisProxyTargetGroup
      Port: 6379
      Protocol: TCP
      LoadBalancerArn: !Ref RedisProxyNLB

  RedisProxyAutoScalingGroup:
    Type: 'AWS::AutoScaling::AutoScalingGroup'
    DependsOn:
      - RedisProxyAutoScalingGroupConfiguration
      - RedisProxyTargetGroup
    Properties:
      TargetGroupARNs:
        - !Ref RedisProxyTargetGroup
      VPCZoneIdentifier: !Split [',', !Join [',', !Ref RedisProxySubnets]]
      DesiredCapacity: !Ref RedisClusterProxyDefaultInstanceCount
      HealthCheckGracePeriod: 100
      LaunchConfigurationName: !Ref RedisProxyAutoScalingGroupConfiguration
      MinSize: !Ref RedisClusterProxyMinimalInstanceCount
      MaxSize: !Ref RedisClusterProxyMaximumInstanceCount
      Tags:
        - Key: Name
          Value: !Ref RedisProxyClusterName
          PropagateAtLaunch: true
  
Parameters:

  RedisClusterProxyNodeConnCount:
    Type: Number
    Description: Set Redis Cluster Proxy node connection count
    Default: 2
  RedisClusterProxyAutoScallingCPUThreshold:
    Type: Number
    Description: Set CPU threshold for redis proxy cluster scale up
    Default: 60
  RedisClusterProxyDefaultInstanceCount:
    Type: Number
    Description: Set initial size of EC2 instances for redis proxy cluster
    Default: 4
  RedisClusterProxyMinimalInstanceCount:
    Type: Number
    Description: Set minimize size of EC2 instances for redis proxy cluster
    Default: 4
  RedisClusterProxyMaximumInstanceCount:
    Type: Number
    Description: Set maximum size of EC2 instances for redis proxy cluster
    Default: 8
  RedisClusterProxyInstanceType:
    Type: String
    Default: c5.large
    AllowedValues:
      - c5.2xlarge
      - c5.xlarge
      - c5.large
    Description: Enter EC2 type for redis cluster proxy.
  RedisProxyClusterName:
    Type: String
    Description: Specific Redis Proxy Name
  RedisClusterConfigurationEndPoint:
    Type: String
    Description: Specific corresponding Redis Cluster Configuration Endpoint
  RedisProxyVPC:
    Type: 'AWS::EC2::VPC::Id'
    Description: Choose one valid VPC for Redis Proxy
  RedisProxySubnets:
    Type: 'List<AWS::EC2::Subnet::Id>'
    Description: Choose one or more valid subnet for Redis Proxy
  RedisProxyKey:
    Type: 'AWS::EC2::KeyPair::KeyName'
    Description: Select the key pair for those EC2 instance
  RedisProxySecurityGroup:
    Type: 'AWS::EC2::SecurityGroup::Id'
    Description: Choose Security Group for this cloudformation

Outputs:
  RedisProxyNLBDNSName:
    Description: The DNSName of the Redis Proxy NLB load balancer
    Value: !GetAtt RedisProxyNLB.DNSName

~ vim overlord-prometheus-metrics.yaml
apiVersion: v1
kind: Service
metadata:
  annotations:
    field.cattle.io/ipAddresses: '["10.170.18.138", "10.170.18.4", "10.170.21.101", "10.170.21.112"]' # overlord proxy private IP list
    prometheus.io/path: /metrics
    prometheus.io/port: "8080"
    prometheus.io/scrape: "true"
  labels:
    app: overload-metrics
  name: expose-metrics-overload-metrics
  namespace: prometheus
spec:
  clusterIP: None
  ports:
  - name: http-metrics
    port: 8080
    protocol: TCP
    targetPort: 8080
  sessionAffinity: None
  type: ClusterIP