docker-compose 多节点部署 HOST 模式，每个节点部署 1 FE 1 BE。
配合 keepalived 实现主备集群

前期环境准备

软件环境

软件	版本
Docker	20.0 及以上
docker-compose	20.1 及以上

硬件环境

配置类型	硬件信息	最大运行集群规模
最低配置	2C 4G	1FE 1BE
推荐配置	4C 16G	3FE 3BE

在宿主机执行如下命令

sysctl -w vm.max_map_count=2000000

一、机器规划

准备三台机器，在每台机器上，计划各安装一个 be 和 fe

机器IP	doris节点	节点端口
192.168.9.60	doris-be/doris-fe	8040/8030 、9030
192.168.9.61	doris-be/doris-fe	8040/8030 、9030
192.168.9.62	doris-be/doris-fe	8040/8030 、9030

二、创建 nfs 目录

在 nfs 服务的机器上创建挂载目录

创建挂载目录脚本，脚本内容如下

#!/bin/bash

# 创建挂载目录
mkdir /data/nfs/doris/0/fe -p
mkdir /data/nfs/doris/0/be/storage -p
mkdir /data/nfs/doris/1/fe -p
mkdir /data/nfs/doris/1/be/storage -p
mkdir /data/nfs/doris/2/fe -p
mkdir /data/nfs/doris/2/be/storage -p
chmod 777 /data/nfs -R

cat >>/etc/exports <<EOF
/data/nfs/doris/0/fe *(rw,sync,no_root_squash)
/data/nfs/doris/0/be/storage *(rw,sync,no_root_squash)
/data/nfs/doris/1/fe *(rw,sync,no_root_squash)
/data/nfs/doris/1/be/storage *(rw,sync,no_root_squash)
/data/nfs/doris/2/fe *(rw,sync,no_root_squash)
/data/nfs/doris/2/be/storage *(rw,sync,no_root_squash)
EOF

exportfs -r

三、创建 be.conf 和 fe.conf 文件

在三台机器的 /data/nfs/doris/fe/conf 下创建 fe.conf 文件，内容如下：
其中 min_replication_num_per_tablet 代表几个 be 副本

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#####################################################################
## The uppercase properties are read and exported by bin/start_fe.sh.
## To see all Frontend configurations,
## see fe/src/org/apache/doris/common/Config.java
#####################################################################
CUR_DATE=`date +%Y%m%d-%H%M%S`
# the output dir of stderr and stdout
LOG_DIR = ${DORIS_HOME}/log
JAVA_OPTS="-server -Xmx20g -Xmn10g -Xms6g -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=15 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=65 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$DATE"
# For jdk 9+, this JAVA_OPTS will be used as default JVM options
JAVA_OPTS_FOR_JDK_9="-Xss4m -Xmx8192m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$CUR_DATE:time"
enable_fqdn_mode = true
##
## the lowercase properties are read by main program.
##
# INFO, WARN, ERROR, FATAL
sys_log_level = INFO
# NORMAL, BRIEF, ASYNC
sys_log_mode = NORMAL
# store metadata, must be created before start FE.
# Default value is ${DORIS_HOME}/doris-meta
# meta_dir = ${DORIS_HOME}/doris-meta
# Default dirs to put jdbc drivers,default value is ${DORIS_HOME}/jdbc_drivers
# jdbc_drivers_dir = ${DORIS_HOME}/jdbc_drivers
http_port = 8030
rpc_port = 9020
query_port = 9030
edit_log_port = 9010
min_replication_num_per_tablet = 1
# Choose one if there are more than one ip except loopback address.
# Note that there should at most one ip match this list.
# If no ip match this rule, will choose one randomly.
# use CIDR format, e.g. 10.10.10.0/24 or IP format, e.g. 10.10.10.1
# Default value is empty.
# priority_networks = 10.10.10.0/24;192.168.0.0/16
# Advanced configurations
# log_roll_size_mb = 1024
# sys_log_dir = ${DORIS_HOME}/log
# sys_log_roll_num = 10
# sys_log_verbose_modules = org.apache.doris
# audit_log_dir = ${DORIS_HOME}/log
# audit_log_modules = slow_query, query
# audit_log_roll_num = 10
# meta_delay_toleration_second = 10
# qe_max_connection = 1024
# qe_query_timeout_second = 300
# qe_slow_log_ms = 5000

在三台机器的 /data/nfs/doris/be/conf 下创建 be.conf 文件，内容如下：

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
CUR_DATE=`date +%Y%m%d-%H%M%S`
PPROF_TMPDIR="$DORIS_HOME/log/"
JAVA_OPTS="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -DJDBC_MIN_POOL=1 -DJDBC_MAX_POOL=100 -DJDBC_MAX_IDLE_TIME=300000 -DJDBC_MAX_WAIT_TIME=5000"
# For jdk 9+, this JAVA_OPTS will be used as default JVM options
JAVA_OPTS_FOR_JDK_9="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -DJDBC_MIN_POOL=1 -DJDBC_MAX_POOL=100 -DJDBC_MAX_IDLE_TIME=300000 -DJDBC_MAX_WAIT_TIME=5000"
# since 1.2, the JAVA_HOME need to be set to run BE process.
# JAVA_HOME=/path/to/jdk/
# https://github.com/apache/doris/blob/master/docs/zh-CN/community/developer-guide/debug-tool.md#jemalloc-heap-profile
# https://jemalloc.net/jemalloc.3.html
JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,lg_tcache_max:20,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false"
JEMALLOC_PROF_PRFIX=""
enable_fqdn_mode = true
# INFO, WARNING, ERROR, FATAL
sys_log_level = INFO
# ports for admin, web, heartbeat service
be_port = 9060
webserver_port = 8040
heartbeat_service_port = 9050
brpc_port = 8060
# HTTPS configures
enable_https = false
# path of certificate in PEM format.
ssl_certificate_path = "$DORIS_HOME/conf/cert.pem"
# path of private key in PEM format.
ssl_private_key_path = "$DORIS_HOME/conf/key.pem"
# enable auth check
enable_auth = false
# Choose one if there are more than one ip except loopback address.
# Note that there should at most one ip match this list.
# If no ip match this rule, will choose one randomly.
# use CIDR format, e.g. 10.10.10.0/24 or IP format, e.g. 10.10.10.1
# Default value is empty.
# priority_networks = 10.10.10.0/24;192.168.0.0/16
# data root path, separate by ';'
# you can specify the storage medium of each root path, HDD or SSD
# you can add capacity limit at the end of each root path, separate by ','
# eg:
# storage_root_path = /home/disk1/doris.HDD,50;/home/disk2/doris.SSD,1;/home/disk2/doris
# /home/disk1/doris.HDD, capacity limit is 50GB, HDD;
# /home/disk2/doris.SSD, capacity limit is 1GB, SSD;
# /home/disk2/doris, capacity limit is disk capacity, HDD(default)
#
# you also can specify the properties by setting '<property>:<value>', separate by ','
# property 'medium' has a higher priority than the extension of path
#
# Default value is ${DORIS_HOME}/storage, you should create it by hand.
# storage_root_path = ${DORIS_HOME}/storage
# Default dirs to put jdbc drivers,default value is ${DORIS_HOME}/jdbc_drivers
# jdbc_drivers_dir = ${DORIS_HOME}/jdbc_drivers
# Advanced configurations
# sys_log_dir = ${DORIS_HOME}/log
# sys_log_roll_mode = SIZE-MB-1024
# sys_log_roll_num = 10
# sys_log_verbose_modules = *
# log_buffer_level = -1
# palo_cgroups
string_type_length_soft_limit_bytes = 104857600
disable_storage_page_cache = false

四、编辑 docker-compose_1fe1be.yml 文件

在三台机器的 /etc/hosts 文件中加上下面 yml 配置中的 hostname

echo "192.168.9.60 fe-01
192.168.9.60 be-01
192.168.9.61 fe-02
192.168.9.61 be-02
192.168.9.62 fe-03
192.168.9.62 be-03" >> /etc/hosts

然后三台机器上分别编写 docker-compose_1fe1be.yml 文件，将 be.conf 和 fe.conf 文件挂载到容器中

version: "3"
services:
  doris-fe-01:
    image: selectdb/doris.fe-ubuntu:2.0.3
    container_name: doris-fe-01
    hostname: fe-01
    ports:
      - "8030:8030"
      - "9030:9030"
      - "9010:9010"
    environment:
      - FE_SERVERS=fe1:192.168.9.60:9010,fe2:192.168.9.61:9010,fe3:192.168.9.62:9010
      - FE_ID=1
      - TZ=Asia/Shanghai
    volumes:
      - volume-doris-fe:/opt/apache-doris/fe/doris-meta
      - /data/doris/fe/log/:/opt/apache-doris/fe/log/
      - /data/doris/fe/conf/fe.conf:/opt/apache-doris/fe/conf/fe.conf
    network_mode: host
    deploy:
      resources:
        limits:
          cpus: '8'
          memory: 32G
        reservations:
          cpus: '4'
          memory: 16G

  doris-be-01:
    image: selectdb/doris.be-ubuntu:2.0.3
    container_name: doris-be-01
    hostname: be-01
    ports:
      - "9050:9050"
      - "8040:8040"
    environment:
      - FE_SERVERS=fe1:192.168.9.60:9010,fe2:192.168.9.61:9010,fe3:192.168.9.62:9010
      - BE_ADDR=192.168.9.60:9050
      - TZ=Asia/Shanghai
    volumes:
      - volume-doris-be-storage:/opt/apache-doris/be/storage
      - /data/doris/be/script/:/docker-entrypoint-initdb.d/
      - /data/doris/be/conf/be.conf:/opt/apache-doris/be/conf/be.conf
    network_mode: host
    depends_on:
      - doris-fe-01
    deploy:
      resources:
        limits:
          cpus: '8'
          memory: 32G
        reservations:
          cpus: '4'
          memory: 16G

volumes:
  volume-doris-be-storage:
    driver_opts:
      type: nfs
      o: addr=192.168.9.62,nolock,soft,rw
      device: ":/data/nfs/doris/0/be/storage"
  volume-doris-fe:
    driver_opts:
      type: nfs
      o: addr=192.168.9.62,nolock,soft,rw
      device: ":/data/nfs/doris/0/fe"

分别在三台机器上启动服务

# 启动
docker-compose -f docker-compose_1fe1be.yml up -d

注意： 从节点的 fe 启动的的时候可能会出现无法加入到 group 的情况，如图

参考资料：https://blog.csdn.net/gj999999/article/details/140217924
我这刚好和参考资料中的解决方法相反，它通过hostname添加不行，我是通过hostname添加解决了

解决：此时需要进入到 master 节点上的 fe 容器中

# 连上 fe
mysql -uroot -P9030 -h 127.0.0.1

先清除掉两个从节点的 fe

# edit_log_port 可以从 fe.conf 中查看，默认为 9010
# 在 MySQL Client 中执行 ALTER SYSTEM 语句
ALTER SYSTEM DROP FOLLOWER "<ip_address>:<edit_log_port>"

ALTER SYSTEM DROP FOLLOWER "192.168.9.61:9010";
ALTER SYSTEM DROP FOLLOWER "192.168.9.62:9010";

再重新添加这两个从节点（通过hostname去添加）

# edit_log_port 可以从 fe.conf 中查看，默认为 9010
# 在 MySQL Client 中执行 ALTER SYSTEM 语句
ALTER SYSTEM ADD FOLLOWER "<fe_hostname>:<edit_log_port>"

ALTER SYSTEM ADD FOLLOWER "fe-02:9010";
ALTER SYSTEM ADD FOLLOWER "fe-03:9010";

再在 master 执行 show frontends\G; 命令查看 fe 状态
Join 和 Alive 都为 true则表示从节点添加成功了

在网页打开 http://{master_ip}:8030，用户名为 root，密码空着登录进去
查看 fe 节点情况，如下图情况则为成功

查看 be 节点情况，如下图情况则为成功

修改密码|创建用户|设置环境变量

进入 master 节点的 fe 容器，登录fe

mysql -h 127.0.0.1 -uroot -P9030

• 修改密码|创建用户|设置环境变量

SET PASSWORD FOR 'root' = PASSWORD('123456');
CREATE USER 'admin'@'%' IDENTIFIED BY '123456';
GRANT ALL ON *.* TO 'admin'@'%';
set global max_allowed_packet = 200*1024*1024;
SET PROPERTY FOR 'root' 'max_user_connections' = '500';
SET PROPERTY FOR 'admin' 'max_user_connections' = '500';

四、安装 keepalived 并设置 VIP

安装keepalived

yum install -y keepalived;
#设置开机启动
systemctl enable keepalived
cp /usr/sbin/keepalived /etc/init.d/keepalived
chmod +x /etc/init.d/keepalived

check_proxy_doris.sh

当 doris 挂掉，则返回1，keepalived 权重降低，主节点被抢占，让另外一台机器接手

vi /etc/keepalived/check_proxy_doris.sh

#!/bin/bash
counter=$(netstat -na|grep "LISTEN"|grep "9030"|wc -l)
if [ "${counter}" -eq 0 ]; then
  exit 1
else
  exit 0
fi

确保这个脚本有执行权限，并且正确地配置了监听端口。如果监控的服务端口不可用，脚本将返回非零值，这将导致Keepalived认为本地服务器失效，并开始VIP转移的过程。

chmod +x /etc/keepalived/check_proxy_doris.sh

keepalived.conf

编辑 keepalived 配置文件 /etc/keepalived/keepalived.conf

参考资料：https://blog.csdn.net/cnskylee/article/details/131591733

vi /etc/keepalived/keepalived.conf

• mater机器配置内容

! Configuration File for keepalived
global_defs {
   # 指定router_id
   router_id ha01
   script_user root
}
vrrp_script check_haproxy_doris {
   # 定义脚本
   script "/etc/keepalived/check_haproxy_doris.sh"
   # 脚本执行间隔，每2s检测一次
   interval 2
}
vrrp_instance VI_2 {
   # 在ha1、ha2、ha3都为 BACKUP
   state BACKUP
   # 设置为不抢占，m1挂了，m2接管VIP,m1重启不会自动抢回VIP
   nopreempt
   # 具体根据网卡来
   interface bond0
   garp_master_delay 10
   smtp_alert
   # 指定虚拟路由器ID, ha1和ha2和h3此值必须相同
   virtual_router_id 67
   # 在ha2上为80、h3上为60
   priority 100
   # 本机ip
   unicast_src_ip 192.168.9.60
   unicast_peer {   
       #对端ip
       192.168.9.61
       192.168.9.62
   }
   advert_int 1
   authentication {
       auth_type PASS
       # 指定验证密码, ha1和ha2和h3此值必须相同
       auth_pass 123456
   }
   virtual_ipaddress {
        # 指定VIP, ha1和ha2和h3此值必须相同，这里可以使用
        192.168.9.80 dev bond0 label bond0:2
   }
   track_script {
       # 调用上面定义的脚本
       check_haproxy_doris
   }
}

• node1机器配置内容

! Configuration File for keepalived
global_defs {
   # 指定router_id
   router_id ha02
   script_user root
}
vrrp_script check_haproxy_doris {
   # 定义脚本
   script "/etc/keepalived/check_haproxy_doris.sh"
   # 脚本执行间隔，每2s检测一次
   interval 2
}
vrrp_instance VI_2 {
   # 在ha1、ha2、ha3都为 BACKUP
   state BACKUP
   # 设置为不抢占，m1挂了，m2接管VIP,m1重启不会自动抢回VIP
   nopreempt
   # 具体根据网卡来
   interface bond0
   garp_master_delay 10
   smtp_alert
   # 指定虚拟路由器ID, ha1和ha2和h3此值必须相同
   virtual_router_id 67
   # 在ha2上为80、h3上为60
   priority 80
   # 本机ip
   unicast_src_ip 192.168.9.61
   unicast_peer {   
       #对端ip
       192.168.9.60
       192.168.9.62
   }
   advert_int 1
   authentication {
       auth_type PASS
       # 指定验证密码, ha1和ha2和h3此值必须相同
       auth_pass 123456
   }
   virtual_ipaddress {
        # 指定VIP, ha1和ha2和h3此值必须相同，这里可以使用
        192.168.9.80 dev bond0 label bond0:2
   }
   track_script {
       # 调用上面定义的脚本
       check_haproxy_doris
   }
}

• node2机器配置内容

! Configuration File for keepalived
global_defs {
   # 指定router_id
   router_id ha03
   script_user root
}
vrrp_script check_haproxy_doris {
   # 定义脚本
   script "/etc/keepalived/check_haproxy_doris.sh"
   # 脚本执行间隔，每2s检测一次
   interval 2
}
vrrp_instance VI_2 {
   # 在ha1、ha2、ha3都为 BACKUP
   state BACKUP
   # 设置为不抢占，m1挂了，m2接管VIP,m1重启不会自动抢回VIP
   nopreempt
   # 具体根据网卡来
   interface bond0
   garp_master_delay 10
   smtp_alert
   # 指定虚拟路由器ID, ha1和ha2和h3此值必须相同
   virtual_router_id 67
   # 在ha2上为80、h3上为60
   priority 60
   # 本机ip
   unicast_src_ip 192.168.9.62
   unicast_peer {   
       #对端ip
       192.168.9.61
       192.168.9.60
   }
   advert_int 1
   authentication {
       auth_type PASS
       # 指定验证密码, ha1和ha2和h3此值必须相同
       auth_pass 123456
   }
   virtual_ipaddress {
        # 指定VIP, ha1和ha2和h3此值必须相同，这里可以使用
        192.168.9.80 dev bond0 label bond0:2
   }
   track_script {
       # 调用上面定义的脚本
       check_haproxy_doris
   }
}

启动keepalived

systemctl start keepalived

查看keepalived状态

systemctl status keepalived

验证使用 VIP 来连接 doris 数据库

参考资料

Doris 官方部署文档
Doris 集群安装部署
Keepalived 搭建三节点、抢占式的Keepalived主备集群